Compare commits
21 Commits
db73b51e0d
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
| 7545b046af | |||
| cf9082ff72 | |||
| 2afcb861e4 | |||
| b98c36acb7 | |||
| a40cc67c38 | |||
| b632f37e3d | |||
| 6e77cde4cc | |||
| 6c045999f2 | |||
| 10672f3816 | |||
| 3eb2d5ffcb | |||
| 2d99c39604 | |||
| 508708b039 | |||
| ebc630b231 | |||
| e3505855c1 | |||
| c4c03dde60 | |||
| 2cce0fcde1 | |||
| 456ae3545b | |||
| bb49578892 | |||
| 3fc29e3a86 | |||
| 7336f9c95e | |||
| 554beb6b48 |
10
.gitignore
vendored
10
.gitignore
vendored
@@ -7,3 +7,13 @@ ref/
|
|||||||
bin/
|
bin/
|
||||||
.torrent.db*
|
.torrent.db*
|
||||||
*.part
|
*.part
|
||||||
|
|
||||||
|
# media downloaded by running got in the repo tree (never commit content)
|
||||||
|
*.mkv
|
||||||
|
*.mp4
|
||||||
|
*.avi
|
||||||
|
*.mov
|
||||||
|
*.webm
|
||||||
|
*.iso
|
||||||
|
*.mp3
|
||||||
|
*.flac
|
||||||
|
|||||||
BIN
13DL.me_Sho Hagoromo Isu v01-07.rar
Normal file
BIN
13DL.me_Sho Hagoromo Isu v01-07.rar
Normal file
Binary file not shown.
158
README.md
158
README.md
@@ -1,13 +1,10 @@
|
|||||||
# got
|
# got
|
||||||
|
|
||||||
A small, Rob-Pike-style command-line download tool in Go: one program that
|
A command-line download tool in Go. It fetches HTTP(S) files over multiple
|
||||||
downloads HTTP(S) URLs over several connections and BitTorrent
|
connections and downloads BitTorrent magnets and `.torrent` files — one binary,
|
||||||
magnets/torrents, with a focused set of options.
|
a small set of flags. It is a downloader, not a web client: there are no
|
||||||
|
cookie/proxy/auth flags — pass `--header` for a one-off header (auth token,
|
||||||
The design leans on Go's runtime instead of a hand-rolled reactor. There is no
|
cookie, user-agent) when a download needs one.
|
||||||
single-threaded event-poll loop: one goroutine per download blocks on real I/O,
|
|
||||||
a buffered channel of slots bounds concurrency, and `context` carries shutdown.
|
|
||||||
What remains is the data — and, per Pike, the data is the design.
|
|
||||||
|
|
||||||
## Build
|
## Build
|
||||||
|
|
||||||
@@ -15,148 +12,63 @@ What remains is the data — and, per Pike, the data is the design.
|
|||||||
go build -o got .
|
go build -o got .
|
||||||
```
|
```
|
||||||
|
|
||||||
Requires Go 1.25+. The BitTorrent engine is
|
Needs Go 1.25+.
|
||||||
[`anacrolix/torrent`](https://github.com/anacrolix/torrent); everything else is
|
|
||||||
the standard library plus `golang.org/x/{term,time}`.
|
|
||||||
|
|
||||||
## Use
|
## Usage
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
# segmented HTTP download, 16 connections, into ./downloads
|
# HTTP download: parallel out of the box (5 connections); -s turns it up
|
||||||
got -x16 -s16 -d downloads https://example.com/big.iso
|
got https://example.com/big.iso
|
||||||
|
got -s16 https://example.com/big.iso
|
||||||
|
|
||||||
# resume an interrupted download
|
# resume an interrupted download
|
||||||
got -c -x16 https://example.com/big.iso
|
got -c https://example.com/big.iso
|
||||||
|
|
||||||
# the same file from several mirrors at once (one file, connections spread
|
# one file from several mirrors (segments spread across the servers,
|
||||||
# across servers, a dead mirror falls over); -Z downloads them separately instead
|
# a dead mirror falls over); -Z downloads them as separate files instead.
|
||||||
got -x4 https://a.example/big.iso https://b.example/big.iso
|
got -s16 https://a.example/big.iso https://b.example/big.iso
|
||||||
|
|
||||||
# a torrent or magnet; seed for 30 minutes after finishing
|
# a torrent or magnet; seed for 30 minutes after finishing
|
||||||
got --seed-time=30 ubuntu.torrent
|
got --seed-time=30 ubuntu.torrent
|
||||||
got 'magnet:?xt=urn:btih:...'
|
got 'magnet:?xt=urn:btih:...'
|
||||||
|
|
||||||
# list the files in a torrent without downloading
|
# many downloads, two at a time
|
||||||
got -S -T some.torrent
|
|
||||||
|
|
||||||
# several downloads at once, two running in parallel
|
|
||||||
got -j2 -i urls.txt
|
got -j2 -i urls.txt
|
||||||
```
|
```
|
||||||
|
|
||||||
Run `got --help` (or `--help=http`, `--help=bittorrent`, ...) for the full
|
Run `got --help` (or `--help=all`) for every option.
|
||||||
option list.
|
|
||||||
|
|
||||||
### Options
|
## Common options
|
||||||
|
|
||||||
A focused subset of options, with familiar short flags and sensible defaults.
|
|
||||||
|
|
||||||
| flag | meaning | default |
|
| flag | meaning | default |
|
||||||
|------|---------|---------|
|
|------|---------|---------|
|
||||||
| `-d, --dir` | output directory | `.` |
|
| `-d, --dir` | output directory | `.` |
|
||||||
| `-o, --out` | output filename (single download) | from URL |
|
| `-o, --out` | output filename | from URL |
|
||||||
| `-i, --input-file` | read downloads line by line (TAB-separated URLs = mirrors; `-` = stdin) | |
|
|
||||||
| `-c, --continue` | resume a partial download | false |
|
| `-c, --continue` | resume a partial download | false |
|
||||||
| `-x, --max-connection-per-server` | connections to one server (1–16) | 1 |
|
| `-s, --split` | connections per download (the speed dial) | 5 |
|
||||||
| `-s, --split` | split a download into N connections | 5 |
|
| `-j, --max-concurrent-downloads` | files downloaded at once | 5 |
|
||||||
| `-Z, --force-sequential` | download each command-line URL separately, not as mirrors | false |
|
|
||||||
| `-k, --min-split-size` | do not split below this size | 20M |
|
|
||||||
| `-j, --max-concurrent-downloads` | parallel downloads | 5 |
|
|
||||||
| `--max-overall-download-limit` | global speed cap | 0 (off) |
|
| `--max-overall-download-limit` | global speed cap | 0 (off) |
|
||||||
| `--max-download-limit` | per-download speed cap | 0 (off) |
|
| `--checksum` | verify the finished file: `TYPE=DIGEST` (sha-256, sha-1, …) | |
|
||||||
| `--retry-wait` | seconds to wait between retries | 0 |
|
| `--seed-time` | minutes to seed after a torrent finishes | by ratio |
|
||||||
| `--load-cookies` / `--save-cookies` | read/write a Netscape `cookies.txt` | |
|
|
||||||
| `--conditional-get` | skip the download if the local file is up to date | false |
|
|
||||||
| `--remote-time` | set the file's mtime from the server | false |
|
|
||||||
| `--checksum` | verify the finished file: `TYPE=DIGEST` (sha-256, sha-1, md5, …) | |
|
|
||||||
| `--dry-run` | check the file is available but do not download it | false |
|
|
||||||
| `--ca-certificate` | verify HTTPS against the CA certificates in FILE (PEM) | |
|
|
||||||
| `--stop` | stop the program after N seconds | 0 (off) |
|
|
||||||
| `--disable-ipv6` | force IPv4-only connections | false |
|
|
||||||
| `-T, --torrent-file` | a `.torrent` file | |
|
|
||||||
| `--seed-time` | minutes to seed after finishing | seed by ratio |
|
|
||||||
| `--seed-ratio` | stop seeding at this ratio | 1.0 |
|
| `--seed-ratio` | stop seeding at this ratio | 1.0 |
|
||||||
| `--listen-port` | port (range) for incoming peers | 6881-6999 |
|
|
||||||
| `--enable-dht` | use the BitTorrent DHT | true |
|
|
||||||
| `--select-file` | fetch only these file indexes (`1,3-5`) | all |
|
|
||||||
| `-S, --show-files` | list torrent files and exit | |
|
| `-S, --show-files` | list torrent files and exit | |
|
||||||
| `--bt-stop-timeout` | give up if no download progress for N s | 0 (off) |
|
| `-q, --quiet` | no progress output | false |
|
||||||
| `--bt-metadata-timeout` | give up if a magnet can't fetch metadata in N s | 60 |
|
|
||||||
| `--save-session` | on exit, write unfinished downloads to FILE | |
|
|
||||||
| `-q, --quiet` | no progress readout | false |
|
|
||||||
|
|
||||||
### Resuming across runs
|
## Resume
|
||||||
|
|
||||||
There is no daemon — resume is a plain file. Point
|
HTTP downloads resume from a small `<file>.got` sidecar — just re-run with `-c`.
|
||||||
`--save-session` and `-i` at the same file and add `-c`: unfinished downloads
|
Torrents resume from the data already on disk. To resume across runs, point
|
||||||
are loaded at start and the still-unfinished ones are written back on exit.
|
`--save-session` and `-i` at the same file:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
got -c --save-session=got.session -i got.session <new urls/magnets...>
|
got -c --save-session=got.session -i got.session <urls/magnets...>
|
||||||
```
|
```
|
||||||
|
|
||||||
On the first run the session file need not exist. HTTP downloads resume from
|
## Notes
|
||||||
their `.got` control file; torrents resume from the data already on disk
|
|
||||||
(re-checked by the engine). A magnet with no reachable peers no longer hangs:
|
|
||||||
`--bt-metadata-timeout` bounds the metadata fetch and defaults to 60s (set it to
|
|
||||||
0 to wait forever).
|
|
||||||
|
|
||||||
## Design
|
- BitTorrent uses [`anacrolix/torrent`](https://github.com/anacrolix/torrent);
|
||||||
|
HTTP uses the standard library.
|
||||||
```
|
- Not implemented: FTP/SFTP, Metalink, and the JSON-RPC server.
|
||||||
main classify URIs, wire options, run, report exit code
|
- Speed caps are global: `--max-overall-download-limit` /
|
||||||
└─ download the Download interface + the scheduler (-j semaphore, ctx)
|
`--max-overall-upload-limit` apply across the whole run.
|
||||||
├─ httpdl []Segment + worker pool + WriteAt + JSON resume sidecar
|
- Standard `HTTP_PROXY` / `HTTPS_PROXY` environment variables are honored.
|
||||||
└─ bt anacrolix/torrent behind the Download interface
|
|
||||||
├─ cli one flat option table -> hand parser -> layered config
|
|
||||||
└─ progress one ticker goroutine, pull-snapshot, \r line redraw
|
|
||||||
```
|
|
||||||
|
|
||||||
The contract is one small interface:
|
|
||||||
|
|
||||||
```go
|
|
||||||
type Download interface {
|
|
||||||
Name() string
|
|
||||||
Run(ctx context.Context) error // blocks in its own goroutine
|
|
||||||
Stat() Stat // a value snapshot, lock-free to read
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
- **HTTP** splits the file into byte-range `Segment`s handed to a pool of
|
|
||||||
workers. Each worker streams its range straight to one shared file with
|
|
||||||
`os.File.WriteAt` — safe for concurrent non-overlapping writes, so there is no
|
|
||||||
shared seek offset and no mutex. Resume is a small JSON sidecar
|
|
||||||
(`<file>.got`) holding per-segment progress plus validators (length +
|
|
||||||
ETag/Last-Modified) so a stale file is never trusted.
|
|
||||||
- **BitTorrent** is `anacrolix/torrent` configured from the CLI options and
|
|
||||||
driven to completion, then seeded for `--seed-time` minutes or up to
|
|
||||||
`--seed-ratio`.
|
|
||||||
- **Options** are one flat `[]Opt` table — the single source of truth for
|
|
||||||
parsing, validation, defaults and `--help`. Layers apply in order:
|
|
||||||
built-in defaults, config file, proxy environment, command line.
|
|
||||||
- **Progress** is a single goroutine that ticks once a second, pulls a snapshot
|
|
||||||
of the running downloads, derives speeds from the change since the last tick,
|
|
||||||
and redraws one line with `\r` + erase-to-end-of-line.
|
|
||||||
|
|
||||||
## Status
|
|
||||||
|
|
||||||
Implemented: segmented HTTP(S) with resume — including a foreign/browser-started
|
|
||||||
partial file (`-c`) — multi-mirror downloads (several URLs, or TAB-grouped `-i`
|
|
||||||
lines, fetch one file with connections spread across servers and fallover; `-Z`
|
|
||||||
to download them separately), retries, rate limits, auto-renaming,
|
|
||||||
`Content-Disposition` naming, conditional GET, cookies
|
|
||||||
(`--load-cookies`/`--save-cookies`), HTTP basic auth, `--remote-time`, whole-file
|
|
||||||
checksum verification (`--checksum`), and a `--dry-run` availability check;
|
|
||||||
BitTorrent download + seeding (magnet and `.torrent`,
|
|
||||||
DHT, trackers, file selection, show-files, metadata-fetch timeout), seeding that
|
|
||||||
stops on whichever of `--seed-time`/`--seed-ratio` comes first, and a listen port
|
|
||||||
chosen from the whole range so a busy port no longer disables BitTorrent;
|
|
||||||
a command-line interface with documented exit codes, config file, live progress,
|
|
||||||
and session save/reload for resume across runs.
|
|
||||||
|
|
||||||
Deferred (not implemented yet): FTP/SFTP, Metalink, and the
|
|
||||||
JSON-RPC server. The architecture leaves room for
|
|
||||||
each — a new protocol is just another `Download`. One known limitation: the
|
|
||||||
BitTorrent engine (`anacrolix/torrent`) applies download/upload rate limits
|
|
||||||
client-wide, so `--max-upload-limit`/`--max-download-limit` act per-run rather
|
|
||||||
than per-torrent when several torrents run at once (`--max-overall-*` are exact);
|
|
||||||
DHT/PEX likewise cannot yet be disabled per private torrent.
|
|
||||||
|
|||||||
141
bt/bt.go
141
bt/bt.go
@@ -9,13 +9,16 @@ package bt
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
missinggo "github.com/anacrolix/missinggo/v2"
|
alog "github.com/anacrolix/log"
|
||||||
"github.com/anacrolix/torrent"
|
"github.com/anacrolix/torrent"
|
||||||
"github.com/anacrolix/torrent/metainfo"
|
"github.com/anacrolix/torrent/metainfo"
|
||||||
"github.com/hanbok/got/download"
|
"github.com/hanbok/got/download"
|
||||||
@@ -26,14 +29,9 @@ import (
|
|||||||
type ClientConfig struct {
|
type ClientConfig struct {
|
||||||
Dir string
|
Dir string
|
||||||
ListenPortSpec string // raw --listen-port spec ("6881-6999,7000"); empty lets the OS choose.
|
ListenPortSpec string // raw --listen-port spec ("6881-6999,7000"); empty lets the OS choose.
|
||||||
DHT bool
|
DHT bool // --enable-dht
|
||||||
MaxPeers int // max peer connections per torrent (0 = library default)
|
|
||||||
OverallDown *rate.Limiter // global down limit (may be nil)
|
OverallDown *rate.Limiter // global down limit (may be nil)
|
||||||
OverallUp *rate.Limiter // global up limit (may be nil)
|
OverallUp *rate.Limiter // global up limit (may be nil)
|
||||||
DownLimit int64 // per-run bytes/s, 0 = unlimited (used only without a global limiter)
|
|
||||||
UpLimit int64
|
|
||||||
UserAgent string
|
|
||||||
DisableIPv6 bool // force IPv4-only (--disable-ipv6)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ParsePorts expands a --listen-port spec into a flat list of candidate ports.
|
// ParsePorts expands a --listen-port spec into a flat list of candidate ports.
|
||||||
@@ -64,57 +62,54 @@ func ParsePorts(spec string) []int {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
for i := a; i <= b; i++ {
|
// Clamp to the valid port space before iterating: a parseable but oversized
|
||||||
if i >= 1 && i <= 65535 {
|
// spec like "1-9999999999" would otherwise spin for ~10^10 iterations and
|
||||||
ports = append(ports, i)
|
// hang startup, even though only [1,65535] can ever be appended.
|
||||||
|
if a < 1 {
|
||||||
|
a = 1
|
||||||
}
|
}
|
||||||
|
if b > 65535 {
|
||||||
|
b = 65535
|
||||||
|
}
|
||||||
|
for i := a; i <= b; i++ {
|
||||||
|
ports = append(ports, i)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return ports
|
return ports
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// quietSlogger filters anacrolix's structured logging down to Error and above,
|
||||||
|
// so the routine webseed/peer/tracker warning chatter (the 403/429 noise) no
|
||||||
|
// longer interleaves with — and corrupts — the live progress display. Genuine
|
||||||
|
// errors still reach w.
|
||||||
|
func quietSlogger(w io.Writer) *slog.Logger {
|
||||||
|
return slog.New(slog.NewTextHandler(w, &slog.HandlerOptions{Level: slog.LevelError}))
|
||||||
|
}
|
||||||
|
|
||||||
// NewClient builds the shared anacrolix client from the run-wide config. When a
|
// NewClient builds the shared anacrolix client from the run-wide config. When a
|
||||||
// listen-port spec is given it tries each candidate port in turn, advancing past
|
// listen-port spec is given it tries each candidate port in turn, advancing past
|
||||||
// any that is already in use, and falls back to an OS-chosen port if none bind,
|
// any that is already in use, and falls back to an OS-chosen port if none bind,
|
||||||
// so a single busy port does not disable BitTorrent entirely.
|
// so a single busy port does not disable BitTorrent entirely. The rate limits
|
||||||
//
|
// and the DHT toggle are client-wide (anacrolix has no per-torrent equivalent),
|
||||||
// Two per-torrent features have no equivalent in anacrolix/torrent v1.61.0,
|
// which is why they live here on the shared client rather than per download.
|
||||||
// which exposes only client-wide knobs; both must therefore be configured run-wide
|
|
||||||
// here rather than attached to an individual Torrent:
|
|
||||||
//
|
|
||||||
// 1. No per-Torrent rate limiter. Throttles are read from the client-wide
|
|
||||||
// DownloadRateLimiter/UploadRateLimiter (no TorrentSpec rate field exists), so
|
|
||||||
// per-torrent --max-download-limit / --max-upload-limit cannot be
|
|
||||||
// honored individually; the effective cap is the shared limiter set below.
|
|
||||||
// 2. No per-Torrent DHT/PEX/LSD toggle for BEP27 private torrents. Those are
|
|
||||||
// client-wide only (NoDHT, DisablePEX, etc.), so a private torrent's leak
|
|
||||||
// prevention must come from a run-wide setting (e.g. --enable-dht=false).
|
|
||||||
func NewClient(cfg ClientConfig) (*torrent.Client, error) {
|
func NewClient(cfg ClientConfig) (*torrent.Client, error) {
|
||||||
c := torrent.NewDefaultClientConfig()
|
c := torrent.NewDefaultClientConfig()
|
||||||
|
// Keep the library's own logging out of the readout. Webseed/peer warnings go
|
||||||
|
// through slog, the rest through the legacy analog logger, so filter both to
|
||||||
|
// Error and above (the analog default is Warning, which is what leaks today).
|
||||||
|
c.Slogger = quietSlogger(os.Stderr)
|
||||||
|
c.Logger = alog.Default.FilterLevel(alog.Error)
|
||||||
c.DataDir = cfg.Dir
|
c.DataDir = cfg.Dir
|
||||||
if c.DataDir == "" {
|
if c.DataDir == "" {
|
||||||
c.DataDir = "."
|
c.DataDir = "."
|
||||||
}
|
}
|
||||||
c.Seed = true
|
c.Seed = true
|
||||||
c.NoDHT = !cfg.DHT
|
c.NoDHT = !cfg.DHT
|
||||||
if cfg.MaxPeers > 0 {
|
if cfg.OverallDown != nil {
|
||||||
c.EstablishedConnsPerTorrent = cfg.MaxPeers
|
|
||||||
}
|
|
||||||
if cfg.UserAgent != "" {
|
|
||||||
c.HTTPUserAgent = cfg.UserAgent
|
|
||||||
}
|
|
||||||
c.DisableIPv6 = cfg.DisableIPv6
|
|
||||||
switch {
|
|
||||||
case cfg.OverallDown != nil:
|
|
||||||
c.DownloadRateLimiter = cfg.OverallDown
|
c.DownloadRateLimiter = cfg.OverallDown
|
||||||
case cfg.DownLimit > 0:
|
|
||||||
c.DownloadRateLimiter = rate.NewLimiter(rate.Limit(cfg.DownLimit), download.LimiterBurst(cfg.DownLimit))
|
|
||||||
}
|
}
|
||||||
switch {
|
if cfg.OverallUp != nil {
|
||||||
case cfg.OverallUp != nil:
|
|
||||||
c.UploadRateLimiter = cfg.OverallUp
|
c.UploadRateLimiter = cfg.OverallUp
|
||||||
case cfg.UpLimit > 0:
|
|
||||||
c.UploadRateLimiter = rate.NewLimiter(rate.Limit(cfg.UpLimit), download.LimiterBurst(cfg.UpLimit))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Candidate ports from the --listen-port spec.
|
// Candidate ports from the --listen-port spec.
|
||||||
@@ -148,10 +143,11 @@ func NewClient(cfg ClientConfig) (*torrent.Client, error) {
|
|||||||
return cl, nil
|
return cl, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// isAddrInUse reports whether err is an "address already in use" bind failure,
|
// isAddrInUse reports whether err is an "address already in use" bind failure.
|
||||||
// matching either missinggo's helper or the raw string the library wraps.
|
// (anacrolix/missinggo's IsAddrInUse is this exact string match, so we just do
|
||||||
|
// it directly rather than carry the import for it.)
|
||||||
func isAddrInUse(err error) bool {
|
func isAddrInUse(err error) bool {
|
||||||
return err != nil && (missinggo.IsAddrInUse(err) || strings.Contains(err.Error(), "address already in use"))
|
return err != nil && strings.Contains(err.Error(), "address already in use")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Options are the per-download settings.
|
// Options are the per-download settings.
|
||||||
@@ -159,8 +155,6 @@ type Options struct {
|
|||||||
SeedTimeSet bool // was --seed-time given?
|
SeedTimeSet bool // was --seed-time given?
|
||||||
SeedTime time.Duration // how long to seed (0 with SeedTimeSet means no seeding)
|
SeedTime time.Duration // how long to seed (0 with SeedTimeSet means no seeding)
|
||||||
SeedRatio float64 // stop seeding at this ratio; checked alongside SeedTime
|
SeedRatio float64 // stop seeding at this ratio; checked alongside SeedTime
|
||||||
StopTimeout time.Duration // abort if no download progress for this long (0 = off)
|
|
||||||
MetaTimeout time.Duration // abort a magnet that can't fetch metadata in this long (0 = off)
|
|
||||||
SelectFiles map[int]bool // 1-based file indexes to fetch; empty = all
|
SelectFiles map[int]bool // 1-based file indexes to fetch; empty = all
|
||||||
CheckIntegrity bool // re-verify data against piece hashes before downloading
|
CheckIntegrity bool // re-verify data against piece hashes before downloading
|
||||||
DryRun bool // fetch metadata only, then stop without downloading (--dry-run)
|
DryRun bool // fetch metadata only, then stop without downloading (--dry-run)
|
||||||
@@ -180,12 +174,17 @@ type Download struct {
|
|||||||
// t is published once Run adds the torrent to the client. It is atomic so
|
// t is published once Run adds the torrent to the client. It is atomic so
|
||||||
// Stat stays lock-free, exactly like httpdl.Stat and bt's own name field.
|
// Stat stays lock-free, exactly like httpdl.Stat and bt's own name field.
|
||||||
t atomic.Pointer[torrent.Torrent]
|
t atomic.Pointer[torrent.Torrent]
|
||||||
|
|
||||||
|
// seeding is closed once Run finishes downloading and transitions to
|
||||||
|
// seed-only, so the engine can drop this torrent from the -j concurrency
|
||||||
|
// count. Created in New, closed exactly once in Run.
|
||||||
|
seeding chan struct{}
|
||||||
}
|
}
|
||||||
|
|
||||||
// New builds a torrent download on the shared client. source is a .torrent path
|
// New builds a torrent download on the shared client. source is a .torrent path
|
||||||
// when isFile is true, otherwise a magnet URI.
|
// when isFile is true, otherwise a magnet URI.
|
||||||
func New(client *torrent.Client, source string, isFile bool, opts Options) *Download {
|
func New(client *torrent.Client, source string, isFile bool, opts Options) *Download {
|
||||||
d := &Download{client: client, source: source, isFile: isFile, opts: opts}
|
d := &Download{client: client, source: source, isFile: isFile, opts: opts, seeding: make(chan struct{})}
|
||||||
if isFile {
|
if isFile {
|
||||||
d.setName(strings.TrimSuffix(filepath.Base(source), ".torrent"))
|
d.setName(strings.TrimSuffix(filepath.Base(source), ".torrent"))
|
||||||
} else {
|
} else {
|
||||||
@@ -210,6 +209,12 @@ func (d *Download) setStatus(s download.Status) {
|
|||||||
atomic.StoreInt32(&d.status, int32(s))
|
atomic.StoreInt32(&d.status, int32(s))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SeedingStarted is closed when this torrent finishes downloading and switches
|
||||||
|
// to seed-only. The engine watches it to release the -j concurrency slot at that
|
||||||
|
// moment: seeding is not downloading, so a seed-only torrent must not hold a slot
|
||||||
|
// that a queued download could use. Implements download.Seeder.
|
||||||
|
func (d *Download) SeedingStarted() <-chan struct{} { return d.seeding }
|
||||||
|
|
||||||
// id is the process-unique identity for this download: the infohash once known,
|
// id is the process-unique identity for this download: the infohash once known,
|
||||||
// otherwise the source string (a pre-metadata magnet has no infohash yet, and
|
// otherwise the source string (a pre-metadata magnet has no infohash yet, and
|
||||||
// two such magnets would otherwise collide on Name).
|
// two such magnets would otherwise collide on Name).
|
||||||
@@ -289,20 +294,24 @@ func (d *Download) Run(ctx context.Context) (err error) {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := d.choose(t); err != nil {
|
// --check-integrity: re-hash the existing on-disk data BEFORE arming the
|
||||||
return err
|
// request loop, so already-good pieces are not re-requested from peers
|
||||||
}
|
// (verify first, then fetch only what is missing).
|
||||||
if d.opts.CheckIntegrity {
|
if d.opts.CheckIntegrity {
|
||||||
if err := t.VerifyDataContext(ctx); err != nil {
|
if err := t.VerifyDataContext(ctx); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if err := d.choose(t); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
if err := d.wait(ctx, t); err != nil {
|
if err := d.wait(ctx, t); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
d.setStatus(download.Seeding)
|
d.setStatus(download.Seeding)
|
||||||
|
close(d.seeding) // free the -j slot: a seed-only torrent no longer downloads
|
||||||
d.seed(ctx, t)
|
d.seed(ctx, t)
|
||||||
d.setStatus(download.Complete)
|
d.setStatus(download.Complete)
|
||||||
return nil
|
return nil
|
||||||
@@ -347,36 +356,35 @@ func (d *Download) choose(t *torrent.Torrent) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// awaitInfo blocks until the torrent metadata arrives. It gives up after
|
// metadataTimeout bounds the magnet metadata-fetch phase. A magnet with no
|
||||||
// --bt-metadata-timeout (default 60s) so a magnet with no reachable peers — a
|
// reachable peers — a dead link, or UDP trackers behind a firewall with no DHT —
|
||||||
// dead link, or UDP trackers behind a firewall with no DHT — fails fast instead
|
// would otherwise hang forever, so awaitInfo gives up after this fixed window.
|
||||||
// of hanging forever. Setting --bt-metadata-timeout=0 disables the timeout and
|
// It is not a user-facing flag — 60s is a generous ceiling that a healthy swarm
|
||||||
// waits indefinitely.
|
// clears in well under a second.
|
||||||
|
const metadataTimeout = 60 * time.Second
|
||||||
|
|
||||||
|
// awaitInfo blocks until the torrent metadata arrives, giving up after
|
||||||
|
// metadataTimeout so a peerless magnet fails fast instead of hanging forever.
|
||||||
func (d *Download) awaitInfo(ctx context.Context, t *torrent.Torrent) error {
|
func (d *Download) awaitInfo(ctx context.Context, t *torrent.Torrent) error {
|
||||||
var deadline <-chan time.Time
|
tm := time.NewTimer(metadataTimeout)
|
||||||
if d.opts.MetaTimeout > 0 {
|
|
||||||
tm := time.NewTimer(d.opts.MetaTimeout)
|
|
||||||
defer tm.Stop()
|
defer tm.Stop()
|
||||||
deadline = tm.C
|
|
||||||
}
|
|
||||||
select {
|
select {
|
||||||
case <-t.GotInfo():
|
case <-t.GotInfo():
|
||||||
return nil
|
return nil
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return ctx.Err()
|
return ctx.Err()
|
||||||
case <-deadline:
|
case <-tm.C:
|
||||||
return fmt.Errorf("timed out fetching metadata after %s", d.opts.MetaTimeout)
|
// Wrap DeadlineExceeded so the exit-code mapping classifies it as a timeout
|
||||||
|
// (2) via errors.Is, rather than matching on the message text.
|
||||||
|
return fmt.Errorf("timed out fetching metadata after %s: %w", metadataTimeout, context.DeadlineExceeded)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// wait blocks until the wanted data is complete, ctx is cancelled, or the
|
// wait blocks until the wanted data is complete or ctx is cancelled, polling
|
||||||
// download stalls past --bt-stop-timeout.
|
// completion a couple of times a second.
|
||||||
func (d *Download) wait(ctx context.Context, t *torrent.Torrent) error {
|
func (d *Download) wait(ctx context.Context, t *torrent.Torrent) error {
|
||||||
tick := time.NewTicker(500 * time.Millisecond)
|
tick := time.NewTicker(500 * time.Millisecond)
|
||||||
defer tick.Stop()
|
defer tick.Stop()
|
||||||
|
|
||||||
last := t.BytesCompleted()
|
|
||||||
lastChange := time.Now()
|
|
||||||
for {
|
for {
|
||||||
if d.complete(t) {
|
if d.complete(t) {
|
||||||
return nil
|
return nil
|
||||||
@@ -385,11 +393,6 @@ func (d *Download) wait(ctx context.Context, t *torrent.Torrent) error {
|
|||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return ctx.Err()
|
return ctx.Err()
|
||||||
case <-tick.C:
|
case <-tick.C:
|
||||||
if n := t.BytesCompleted(); n != last {
|
|
||||||
last, lastChange = n, time.Now()
|
|
||||||
} else if d.opts.StopTimeout > 0 && time.Since(lastChange) > d.opts.StopTimeout {
|
|
||||||
return fmt.Errorf("no progress for %s", d.opts.StopTimeout)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,10 +1,38 @@
|
|||||||
package bt
|
package bt
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"log/slog"
|
||||||
"reflect"
|
"reflect"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// TestQuietSloggerDropsWarnings: the library's warning chatter (the webseed
|
||||||
|
// 403/429 noise) is filtered out so it can't corrupt the live display, while a
|
||||||
|
// genuine error still gets through.
|
||||||
|
func TestQuietSloggerDropsWarnings(t *testing.T) {
|
||||||
|
var buf bytes.Buffer
|
||||||
|
lg := quietSlogger(&buf)
|
||||||
|
|
||||||
|
lg.Warn("webseed request error", "err", "429 Too Many Requests")
|
||||||
|
if buf.Len() != 0 {
|
||||||
|
t.Errorf("warning leaked into output: %q", buf.String())
|
||||||
|
}
|
||||||
|
// anacrolix's webseed code logs via Log(ctx, level, ...); confirm that path is
|
||||||
|
// filtered too, not just the Warn helper.
|
||||||
|
lg.Log(context.Background(), slog.LevelWarn, "another warning")
|
||||||
|
if buf.Len() != 0 {
|
||||||
|
t.Errorf("Log-at-Warn leaked: %q", buf.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
lg.Error("real failure")
|
||||||
|
if !strings.Contains(buf.String(), "real failure") {
|
||||||
|
t.Errorf("error was suppressed, want it kept: %q", buf.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestParsePorts(t *testing.T) {
|
func TestParsePorts(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
spec string
|
spec string
|
||||||
|
|||||||
38
cli/help.go
38
cli/help.go
@@ -89,19 +89,31 @@ func flagLabel(o *Opt) string {
|
|||||||
b.WriteString(" ")
|
b.WriteString(" ")
|
||||||
}
|
}
|
||||||
b.WriteString("--" + o.Long)
|
b.WriteString("--" + o.Long)
|
||||||
switch o.Kind {
|
if m := metavar(o); m != "" {
|
||||||
case Bool:
|
b.WriteString("=" + m)
|
||||||
// no argument shown
|
|
||||||
case Size:
|
|
||||||
b.WriteString("=SIZE")
|
|
||||||
case Int:
|
|
||||||
b.WriteString("=N")
|
|
||||||
case Float:
|
|
||||||
b.WriteString("=RATIO")
|
|
||||||
case Enum:
|
|
||||||
b.WriteString("=" + strings.Join(o.Enum, "|"))
|
|
||||||
default:
|
|
||||||
b.WriteString("=VAL")
|
|
||||||
}
|
}
|
||||||
return b.String()
|
return b.String()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// metavar is the placeholder shown after a value-taking flag, e.g. the "N" in
|
||||||
|
// "--split=N". An explicit Opt.Metavar wins; otherwise it derives from the Kind.
|
||||||
|
// Bool takes no value, so its metavar is empty and no "=ARG" is printed.
|
||||||
|
func metavar(o *Opt) string {
|
||||||
|
if o.Metavar != "" {
|
||||||
|
return o.Metavar
|
||||||
|
}
|
||||||
|
switch o.Kind {
|
||||||
|
case Bool:
|
||||||
|
return ""
|
||||||
|
case Size:
|
||||||
|
return "SIZE"
|
||||||
|
case Int:
|
||||||
|
return "N"
|
||||||
|
case Float:
|
||||||
|
return "RATIO"
|
||||||
|
case Enum:
|
||||||
|
return strings.Join(o.Enum, "|")
|
||||||
|
default:
|
||||||
|
return "VAL"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -45,11 +45,15 @@ func (t Tag) String() string {
|
|||||||
|
|
||||||
// Opt describes one option: its names, value kind, default and help text.
|
// Opt describes one option: its names, value kind, default and help text.
|
||||||
type Opt struct {
|
type Opt struct {
|
||||||
Long string // long name without dashes, e.g. "max-connection-per-server"
|
Long string // long name without dashes, e.g. "max-concurrent-downloads"
|
||||||
Short byte // short flag byte, e.g. 'x'; 0 if none
|
Short byte // short flag byte, e.g. 'x'; 0 if none
|
||||||
Kind Kind
|
Kind Kind
|
||||||
Default string
|
Default string
|
||||||
Help string
|
Help string
|
||||||
|
// Metavar overrides the "=PLACEHOLDER" shown after the flag in --help. Empty
|
||||||
|
// means derive it from Kind (N, SIZE, RATIO, ...). Set it only when the
|
||||||
|
// Kind-derived default would mislead, e.g. seed-time is minutes, not a ratio.
|
||||||
|
Metavar string
|
||||||
Tag Tag
|
Tag Tag
|
||||||
Enum []string // valid values when Kind == Enum
|
Enum []string // valid values when Kind == Enum
|
||||||
// Min is the inclusive lower Int/Size bound. The zero value enforces a
|
// Min is the inclusive lower Int/Size bound. The zero value enforces a
|
||||||
@@ -60,68 +64,44 @@ type Opt struct {
|
|||||||
Max int64
|
Max int64
|
||||||
}
|
}
|
||||||
|
|
||||||
// options is the single source of truth: a focused, practical subset.
|
// options is the single source of truth: a focused, practical subset. The Tag
|
||||||
|
// decides which --help group an option shows under; the Basic group is the bare
|
||||||
|
// `got --help` and is kept to the handful of flags that change WHAT happens, not
|
||||||
|
// how fast. Every other flag still works — it just lives behind --help=<group>.
|
||||||
var options = []Opt{
|
var options = []Opt{
|
||||||
// --- basic ---
|
// --- basic --- the outcome-changing core shown by a bare `got --help`.
|
||||||
{Long: "dir", Short: 'd', Kind: Str, Help: "directory to store downloaded files", Tag: Basic},
|
{Long: "dir", Short: 'd', Kind: Str, Help: "directory to store downloaded files", Tag: Basic},
|
||||||
{Long: "out", Short: 'o', Kind: Str, Help: "output filename for the download", Tag: Basic},
|
{Long: "out", Short: 'o', Kind: Str, Help: "output filename for the download", Tag: Basic},
|
||||||
{Long: "input-file", Short: 'i', Kind: Str, Help: "read URIs line by line from FILE (- for stdin)", Tag: Basic},
|
{Long: "input-file", Short: 'i', Kind: Str, Help: "read URIs line by line from FILE (- for stdin)", Tag: Basic},
|
||||||
|
{Long: "max-concurrent-downloads", Short: 'j', Kind: Int, Default: "5", Min: 1, Help: "how many files to download at once (not connections within one file)", Tag: Basic},
|
||||||
{Long: "continue", Short: 'c', Kind: Bool, Default: "false", Help: "resume a partially downloaded file", Tag: Basic},
|
{Long: "continue", Short: 'c', Kind: Bool, Default: "false", Help: "resume a partially downloaded file", Tag: Basic},
|
||||||
{Long: "max-concurrent-downloads", Short: 'j', Kind: Int, Default: "5", Min: 1, Help: "max number of parallel downloads", Tag: Basic},
|
{Long: "split", Short: 's', Kind: Int, Default: "5", Min: 1, Help: "connections per download — the speed dial (default 5; e.g. -s16 for 16)", Tag: Basic},
|
||||||
{Long: "check-integrity", Short: 'V', Kind: Bool, Default: "false", Help: "re-verify torrent data against piece hashes (BitTorrent)", Tag: Basic},
|
|
||||||
{Long: "show-files", Short: 'S', Kind: Bool, Default: "false", Help: "list files in a torrent and exit", Tag: Basic},
|
|
||||||
{Long: "allow-overwrite", Kind: Bool, Default: "false", Help: "overwrite an existing file", Tag: Basic},
|
|
||||||
{Long: "auto-file-renaming", Kind: Bool, Default: "true", Help: "rename file (.1, .2, ...) if it already exists", Tag: Basic},
|
|
||||||
{Long: "max-overall-download-limit", Kind: Size, Default: "0", Help: "global download speed limit (0 = unlimited)", Tag: Basic},
|
{Long: "max-overall-download-limit", Kind: Size, Default: "0", Help: "global download speed limit (0 = unlimited)", Tag: Basic},
|
||||||
{Long: "max-download-limit", Kind: Size, Default: "0", Help: "per-download speed limit (0 = unlimited)", Tag: Basic},
|
{Long: "checksum", Kind: Str, Help: "verify the finished file: TYPE=DIGEST, e.g. sha-256=<hex> (md5, sha-1, sha-224, sha-256, sha-384, sha-512)", Tag: Basic},
|
||||||
|
{Long: "show-files", Short: 'S', Kind: Bool, Default: "false", Help: "list files in a torrent and exit", Tag: Basic},
|
||||||
{Long: "quiet", Short: 'q', Kind: Bool, Default: "false", Help: "suppress the progress readout", Tag: Basic},
|
{Long: "quiet", Short: 'q', Kind: Bool, Default: "false", Help: "suppress the progress readout", Tag: Basic},
|
||||||
{Long: "human-readable", Kind: Bool, Default: "true", Help: "show sizes as Ki/Mi/Gi", Tag: Basic},
|
|
||||||
|
|
||||||
// --- http ---
|
// --- http ---
|
||||||
{Long: "max-connection-per-server", Short: 'x', Kind: Int, Default: "1", Min: 1, Max: 16, Help: "max connections to one server (1-16)", Tag: HTTP},
|
|
||||||
{Long: "split", Short: 's', Kind: Int, Default: "5", Min: 1, Help: "split a download into N connections; actual connections are min(max-connection-per-server, split), and -x defaults to 1", Tag: HTTP},
|
|
||||||
{Long: "min-split-size", Short: 'k', Kind: Size, Default: "20M", Min: 1 << 20, Max: 1 << 30, Help: "do not split a piece smaller than SIZE (1M-1024M)", Tag: HTTP},
|
|
||||||
{Long: "force-sequential", Short: 'Z', Kind: Bool, Default: "false", Help: "download each command-line URI as its own file instead of mirroring them", Tag: HTTP},
|
{Long: "force-sequential", Short: 'Z', Kind: Bool, Default: "false", Help: "download each command-line URI as its own file instead of mirroring them", Tag: HTTP},
|
||||||
{Long: "max-tries", Short: 'm', Kind: Int, Default: "5", Min: 0, Help: "max retries per segment (0 = unlimited)", Tag: HTTP},
|
{Long: "max-tries", Short: 'm', Kind: Int, Default: "5", Min: 0, Help: "max retries per segment (0 = unlimited)", Tag: HTTP},
|
||||||
{Long: "timeout", Short: 't', Kind: Int, Default: "60", Min: 1, Help: "connection timeout in seconds", Tag: HTTP},
|
{Long: "timeout", Short: 't', Kind: Int, Default: "60", Min: 1, Help: "connection/idle timeout in seconds", Tag: HTTP},
|
||||||
{Long: "header", Kind: List, Help: "append an extra HTTP header (repeatable)", Tag: HTTP},
|
{Long: "header", Kind: List, Help: "append an extra HTTP header (repeatable), e.g. 'Authorization: Bearer …'", Tag: HTTP},
|
||||||
{Long: "user-agent", Short: 'U', Kind: Str, Default: "got/1.0", Help: "HTTP User-Agent", Tag: HTTP},
|
|
||||||
{Long: "referer", Kind: Str, Help: "HTTP Referer header", Tag: HTTP},
|
|
||||||
{Long: "all-proxy", Kind: Str, Help: "proxy for all protocols (host:port)", Tag: HTTP},
|
|
||||||
{Long: "check-certificate", Kind: Bool, Default: "true", Help: "verify the server's TLS certificate", Tag: HTTP},
|
|
||||||
{Long: "ca-certificate", Kind: Str, Help: "verify HTTPS servers against the CA certificates in FILE (PEM)", Tag: HTTP},
|
|
||||||
{Long: "retry-wait", Kind: Int, Default: "0", Min: 0, Max: 600, Help: "seconds to wait between retries (0 = no wait)", Tag: HTTP},
|
|
||||||
{Long: "connect-timeout", Kind: Int, Default: "60", Min: 1, Help: "connection establishment timeout in seconds", Tag: HTTP},
|
|
||||||
{Long: "load-cookies", Kind: Str, Help: "load Cookies from FILE (Netscape/Mozilla format)", Tag: HTTP},
|
|
||||||
{Long: "save-cookies", Kind: Str, Help: "save Cookies to FILE on exit", Tag: HTTP},
|
|
||||||
{Long: "conditional-get", Kind: Bool, Default: "false", Help: "download only if the remote file is newer than the local one", Tag: HTTP},
|
|
||||||
{Long: "remote-time", Kind: Bool, Default: "false", Help: "set the local file's mtime from the server", Tag: HTTP},
|
|
||||||
{Long: "http-user", Kind: Str, Help: "HTTP basic-auth user", Tag: HTTP},
|
|
||||||
{Long: "http-passwd", Kind: Str, Help: "HTTP basic-auth password", Tag: HTTP},
|
|
||||||
{Long: "lowest-speed-limit", Kind: Size, Default: "0", Help: "abort if speed stays below SIZE bytes/sec (0 = off)", Tag: HTTP},
|
|
||||||
{Long: "checksum", Kind: Str, Help: "verify the finished file: TYPE=DIGEST, e.g. sha-256=<hex> (md5, sha-1, sha-224, sha-256, sha-384, sha-512)", Tag: HTTP},
|
|
||||||
|
|
||||||
// --- bittorrent ---
|
// --- bittorrent ---
|
||||||
{Long: "torrent-file", Short: 'T', Kind: Str, Help: "path to a .torrent file", Tag: BitTorrent},
|
{Long: "check-integrity", Short: 'V', Kind: Bool, Default: "false", Help: "re-verify torrent data against piece hashes", Tag: BitTorrent},
|
||||||
{Long: "seed-time", Kind: Float, Help: "minutes to seed after completion (0 = no seeding)", Tag: BitTorrent},
|
{Long: "seed-time", Kind: Float, Metavar: "MIN", Help: "minutes to seed after completion (0 = no seeding)", Tag: BitTorrent},
|
||||||
{Long: "seed-ratio", Kind: Float, Default: "1.0", Help: "stop seeding at this share ratio (0 = unlimited)", Tag: BitTorrent},
|
{Long: "seed-ratio", Kind: Float, Default: "1.0", Help: "stop seeding at this share ratio (0 = unlimited)", Tag: BitTorrent},
|
||||||
{Long: "bt-stop-timeout", Kind: Int, Default: "0", Min: 0, Help: "stop a torrent with no download progress for N seconds (0 = off)", Tag: BitTorrent},
|
|
||||||
{Long: "bt-metadata-timeout", Kind: Int, Default: "60", Min: 0, Help: "stop a magnet that can't fetch metadata in N seconds (0 = off)", Tag: BitTorrent},
|
|
||||||
{Long: "listen-port", Kind: Str, Default: "6881-6999", Help: "TCP port (range) for incoming peers", Tag: BitTorrent},
|
{Long: "listen-port", Kind: Str, Default: "6881-6999", Help: "TCP port (range) for incoming peers", Tag: BitTorrent},
|
||||||
{Long: "enable-dht", Kind: Bool, Default: "true", Help: "enable the BitTorrent DHT", Tag: BitTorrent},
|
{Long: "enable-dht", Kind: Bool, Default: "true", Help: "enable the BitTorrent DHT (turn off for private torrents)", Tag: BitTorrent},
|
||||||
{Long: "bt-max-peers", Kind: Int, Default: "55", Min: 0, Help: "max peers per torrent (0 = unlimited)", Tag: BitTorrent},
|
|
||||||
{Long: "max-overall-upload-limit", Kind: Size, Default: "0", Help: "global upload speed limit (0 = unlimited)", Tag: BitTorrent},
|
{Long: "max-overall-upload-limit", Kind: Size, Default: "0", Help: "global upload speed limit (0 = unlimited)", Tag: BitTorrent},
|
||||||
{Long: "max-upload-limit", Short: 'u', Kind: Size, Default: "0", Help: "per-torrent upload speed limit", Tag: BitTorrent},
|
|
||||||
{Long: "select-file", Kind: Str, Help: "download only these file indexes, e.g. 1,3-5", Tag: BitTorrent},
|
{Long: "select-file", Kind: Str, Help: "download only these file indexes, e.g. 1,3-5", Tag: BitTorrent},
|
||||||
{Long: "follow-torrent", Kind: Bool, Default: "true", Help: "after fetching a .torrent over HTTP, download its content", Tag: BitTorrent},
|
|
||||||
|
|
||||||
// --- advanced ---
|
// --- advanced ---
|
||||||
{Long: "file-allocation", Short: 'a', Kind: Enum, Default: "prealloc", Enum: []string{"none", "prealloc", "trunc", "falloc"}, Help: "how to allocate disk space", Tag: Advanced},
|
{Long: "allow-overwrite", Kind: Bool, Default: "false", Help: "overwrite an existing file instead of renaming the new one", Tag: Advanced},
|
||||||
|
{Long: "auto-file-renaming", Kind: Bool, Default: "true", Help: "rename file (.1, .2, ...) if it already exists", Tag: Advanced},
|
||||||
|
{Long: "file-allocation", Short: 'a', Kind: Enum, Default: "prealloc", Enum: []string{"none", "prealloc", "trunc", "falloc"}, Help: "how to reserve disk space (prealloc and falloc are identical here)", Tag: Advanced},
|
||||||
{Long: "dry-run", Kind: Bool, Default: "false", Help: "check that the file is available but do not download it", Tag: Advanced},
|
{Long: "dry-run", Kind: Bool, Default: "false", Help: "check that the file is available but do not download it", Tag: Advanced},
|
||||||
{Long: "stop", Kind: Int, Default: "0", Min: 0, Help: "stop the program after N seconds (0 = off)", Tag: Advanced},
|
|
||||||
{Long: "disable-ipv6", Kind: Bool, Default: "false", Help: "disable IPv6 (force IPv4-only connections)", Tag: Advanced},
|
|
||||||
{Long: "save-session", Kind: Str, Help: "on exit, write unfinished downloads to FILE (reload with -i)", Tag: Advanced},
|
{Long: "save-session", Kind: Str, Help: "on exit, write unfinished downloads to FILE (reload with -i)", Tag: Advanced},
|
||||||
{Long: "auto-save-interval", Kind: Int, Default: "60", Min: 0, Max: 600, Help: "save the session file every N seconds (0 = only on exit)", Tag: Advanced},
|
|
||||||
{Long: "conf-path", Kind: Str, Help: "path to the config file", Tag: Advanced},
|
{Long: "conf-path", Kind: Str, Help: "path to the config file", Tag: Advanced},
|
||||||
{Long: "no-conf", Kind: Bool, Default: "false", Help: "do not read a config file", Tag: Advanced},
|
{Long: "no-conf", Kind: Bool, Default: "false", Help: "do not read a config file", Tag: Advanced},
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,21 +2,24 @@ package cli
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"math"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Options is the resolved configuration: a flat name->value map plus a record
|
// Options is the resolved configuration: a flat name->value map plus a record
|
||||||
// of which names were explicitly set (so callers can tell a default from a
|
// of which names were explicitly set (so callers can tell a default from a
|
||||||
// chosen value). Values are kept as strings end-to-end and converted only at
|
// chosen value). The raw strings carry the layered override merge; finalize()
|
||||||
// the point of use by the typed getters below.
|
// then parses each one exactly once into typed, and the numeric/bool getters
|
||||||
|
// read that — no value is parsed twice.
|
||||||
type Options struct {
|
type Options struct {
|
||||||
vals map[string]string
|
vals map[string]string
|
||||||
set map[string]bool
|
set map[string]bool
|
||||||
|
typed map[string]any // parsed value per option, filled once by finalize()
|
||||||
}
|
}
|
||||||
|
|
||||||
func newOptions() *Options {
|
func newOptions() *Options {
|
||||||
return &Options{vals: map[string]string{}, set: map[string]bool{}}
|
return &Options{vals: map[string]string{}, set: map[string]bool{}, typed: map[string]any{}}
|
||||||
}
|
}
|
||||||
|
|
||||||
// IsSet reports whether name was given on the command line or in the config
|
// IsSet reports whether name was given on the command line or in the config
|
||||||
@@ -26,47 +29,30 @@ func (o *Options) IsSet(name string) bool { return o.set[name] }
|
|||||||
// Str returns the raw string value (empty if unset and no default).
|
// Str returns the raw string value (empty if unset and no default).
|
||||||
func (o *Options) Str(name string) string { return o.vals[name] }
|
func (o *Options) Str(name string) string { return o.vals[name] }
|
||||||
|
|
||||||
// boolWords is the single accepted vocabulary for boolean options, mapping each
|
// boolWords is the accepted vocabulary for boolean options: exactly true and
|
||||||
// recognised spelling to its truth value. The Bool reader, truthy, and
|
// false, nothing else. boolWord is the single consult point — the Bool reader,
|
||||||
// validate() all consult it, so exactly the words that validate are honoured
|
// validate(), and the no-conf bootstrap all go through it — so exactly the words
|
||||||
// (no "accepted by the reader but rejected by validate" surprises like --x=on).
|
// that validate are honoured.
|
||||||
var boolWords = map[string]bool{
|
var boolWords = map[string]bool{
|
||||||
"true": true, "yes": true, "1": true, "on": true,
|
"true": true,
|
||||||
"false": false, "no": false, "0": false, "off": false,
|
"false": false,
|
||||||
}
|
}
|
||||||
|
|
||||||
// boolWord reports a value's truth and whether it is a recognised boolean word.
|
// boolWord reports a value's truth and whether it is a recognised boolean word.
|
||||||
|
// Whitespace is trimmed, but case is significant: "True"/"TRUE" are rejected, so
|
||||||
|
// the match is against exactly "true"/"false".
|
||||||
func boolWord(s string) (val, ok bool) {
|
func boolWord(s string) (val, ok bool) {
|
||||||
val, ok = boolWords[strings.ToLower(strings.TrimSpace(s))]
|
val, ok = boolWords[strings.TrimSpace(s)]
|
||||||
return val, ok
|
return val, ok
|
||||||
}
|
}
|
||||||
|
|
||||||
// Bool reports whether the value is a truthy boolean word.
|
// Bool, Int, Float and Size read the value finalize() already parsed and stored
|
||||||
func (o *Options) Bool(name string) bool {
|
// in typed; an unset option (no entry) reads as the zero value. The conversion
|
||||||
val, _ := boolWord(o.vals[name])
|
// happened once, at finalize, so these never re-parse a string.
|
||||||
return val
|
func (o *Options) Bool(name string) bool { v, _ := o.typed[name].(bool); return v }
|
||||||
}
|
func (o *Options) Int(name string) int { v, _ := o.typed[name].(int64); return int(v) }
|
||||||
|
func (o *Options) Float(name string) float64 { v, _ := o.typed[name].(float64); return v }
|
||||||
// Int returns the value as an int, or 0 if empty/invalid.
|
func (o *Options) Size(name string) int64 { v, _ := o.typed[name].(int64); return v }
|
||||||
func (o *Options) Int(name string) int { return int(o.Int64(name)) }
|
|
||||||
|
|
||||||
// Int64 returns the value as an int64, or 0 if empty/invalid.
|
|
||||||
func (o *Options) Int64(name string) int64 {
|
|
||||||
n, _ := strconv.ParseInt(strings.TrimSpace(o.vals[name]), 10, 64)
|
|
||||||
return n
|
|
||||||
}
|
|
||||||
|
|
||||||
// Float returns the value as a float64, or 0 if empty/invalid.
|
|
||||||
func (o *Options) Float(name string) float64 {
|
|
||||||
f, _ := strconv.ParseFloat(strings.TrimSpace(o.vals[name]), 64)
|
|
||||||
return f
|
|
||||||
}
|
|
||||||
|
|
||||||
// Size returns a byte count parsed from a value like "20M" or "512K".
|
|
||||||
func (o *Options) Size(name string) int64 {
|
|
||||||
n, _ := parseSize(o.vals[name])
|
|
||||||
return n
|
|
||||||
}
|
|
||||||
|
|
||||||
// List returns a repeatable option's accumulated values.
|
// List returns a repeatable option's accumulated values.
|
||||||
func (o *Options) List(name string) []string {
|
func (o *Options) List(name string) []string {
|
||||||
@@ -105,5 +91,10 @@ func parseSize(s string) (int64, error) {
|
|||||||
if n < 0 {
|
if n < 0 {
|
||||||
return 0, fmt.Errorf("negative size %q", s)
|
return 0, fmt.Errorf("negative size %q", s)
|
||||||
}
|
}
|
||||||
|
// Reject a value whose unit multiply would overflow int64 and silently wrap to
|
||||||
|
// a bogus (positive or negative) byte count.
|
||||||
|
if mult > 1 && n > math.MaxInt64/mult {
|
||||||
|
return 0, fmt.Errorf("size %q too large", s)
|
||||||
|
}
|
||||||
return n * mult, nil
|
return n * mult, nil
|
||||||
}
|
}
|
||||||
|
|||||||
123
cli/parse.go
123
cli/parse.go
@@ -27,8 +27,8 @@ type Result struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Parse resolves args into options and URIs, applying the layered override
|
// Parse resolves args into options and URIs, applying the layered override
|
||||||
// order: built-in defaults, then the config file, then proxy
|
// order: built-in defaults, then the config file, then the command line
|
||||||
// environment variables, then the command line (last wins).
|
// (last wins).
|
||||||
func Parse(args []string) (*Result, error) {
|
func Parse(args []string) (*Result, error) {
|
||||||
cmdVals, uris, action, helpTag, err := parseArgs(args)
|
cmdVals, uris, action, helpTag, err := parseArgs(args)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -41,23 +41,47 @@ func Parse(args []string) (*Result, error) {
|
|||||||
opts := newOptions()
|
opts := newOptions()
|
||||||
applyDefaults(opts)
|
applyDefaults(opts)
|
||||||
|
|
||||||
if !truthy(cmdVals["no-conf"]) {
|
if noConf, _ := boolWord(cmdVals["no-conf"]); !noConf {
|
||||||
explicit := cmdVals["conf-path"]
|
explicit := cmdVals["conf-path"]
|
||||||
path, fromUser := confPath(explicit)
|
path, fromUser := confPath(explicit)
|
||||||
if err := applyConfig(opts, path, fromUser); err != nil {
|
if err := applyConfig(opts, path, fromUser); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
applyEnv(opts)
|
|
||||||
|
|
||||||
for name, val := range cmdVals {
|
for name, val := range cmdVals {
|
||||||
if err := set(opts, name, val); err != nil {
|
if err := set(opts, name, val); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if err := opts.finalize(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
return &Result{Action: Run, Options: opts, URIs: uris}, nil
|
return &Result{Action: Run, Options: opts, URIs: uris}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// finalize parses every resolved value once into its typed form, validating it
|
||||||
|
// in the same pass; the numeric/bool getters then just read the result. It runs
|
||||||
|
// after all layering, so each option is converted exactly once, from its final
|
||||||
|
// (last-wins) string — not once per layer and again per read.
|
||||||
|
func (o *Options) finalize() error {
|
||||||
|
for i := range options {
|
||||||
|
opt := &options[i]
|
||||||
|
raw, ok := o.vals[opt.Long]
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
v, err := parseValue(opt, raw)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if v != nil {
|
||||||
|
o.typed[opt.Long] = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// parseArgs walks the argument list once. It recognises --long, --long=val,
|
// parseArgs walks the argument list once. It recognises --long, --long=val,
|
||||||
// -x, -xval, -x val and short bool bundles like -cq. Like getopt_long, it
|
// -x, -xval, -x val and short bool bundles like -cq. Like getopt_long, it
|
||||||
// permutes: a non-flag token becomes a URI but scanning continues so flags may
|
// permutes: a non-flag token becomes a URI but scanning continues so flags may
|
||||||
@@ -138,14 +162,23 @@ func parseArgs(args []string) (vals map[string]string, uris []string, action Act
|
|||||||
// accumulate stores a value, joining repeatable List options with newlines.
|
// accumulate stores a value, joining repeatable List options with newlines.
|
||||||
func accumulate(vals map[string]string, o *Opt, val string) {
|
func accumulate(vals map[string]string, o *Opt, val string) {
|
||||||
if o.Kind == List {
|
if o.Kind == List {
|
||||||
if prev, ok := vals[o.Long]; ok {
|
vals[o.Long] = appendList(vals[o.Long], val)
|
||||||
vals[o.Long] = prev + "\n" + val
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
|
||||||
vals[o.Long] = val
|
vals[o.Long] = val
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// appendList is the single home of the List newline-join rule, shared by the
|
||||||
|
// raw-parse layer (accumulate) and the resolved layer (set). An empty prev (the
|
||||||
|
// option not yet seen) yields val unchanged, so the first value carries no
|
||||||
|
// leading newline.
|
||||||
|
func appendList(prev, val string) string {
|
||||||
|
if prev == "" {
|
||||||
|
return val
|
||||||
|
}
|
||||||
|
return prev + "\n" + val
|
||||||
|
}
|
||||||
|
|
||||||
func applyDefaults(o *Options) {
|
func applyDefaults(o *Options) {
|
||||||
for i := range options {
|
for i := range options {
|
||||||
opt := &options[i]
|
opt := &options[i]
|
||||||
@@ -191,86 +224,73 @@ func applyConfig(o *Options, path string, explicit bool) error {
|
|||||||
return sc.Err()
|
return sc.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
// applyEnv folds the conventional proxy environment variables into all-proxy
|
// set stores a raw value, marking the option as explicitly set. Validation and
|
||||||
// unless it was already set explicitly.
|
// conversion happen later, once, in finalize(); set only records the string.
|
||||||
func applyEnv(o *Options) {
|
|
||||||
if o.set["all-proxy"] {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
for _, key := range []string{"all_proxy", "https_proxy", "http_proxy"} {
|
|
||||||
if v := os.Getenv(key); v != "" {
|
|
||||||
o.vals["all-proxy"] = v
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// set validates a value against its option descriptor and stores it, marking
|
|
||||||
// the option as explicitly set.
|
|
||||||
func set(o *Options, name, val string) error {
|
func set(o *Options, name, val string) error {
|
||||||
opt := byLong[name]
|
opt := byLong[name]
|
||||||
if opt == nil {
|
if opt == nil {
|
||||||
return fmt.Errorf("unknown option %q", name)
|
return fmt.Errorf("unknown option %q", name)
|
||||||
}
|
}
|
||||||
if err := validate(opt, val); err != nil {
|
if opt.Kind == List && o.set[name] {
|
||||||
return err
|
val = appendList(o.vals[name], val)
|
||||||
}
|
|
||||||
if opt.Kind == List {
|
|
||||||
if prev, ok := o.vals[name]; ok && o.set[name] {
|
|
||||||
val = prev + "\n" + val
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
o.vals[name] = val
|
o.vals[name] = val
|
||||||
o.set[name] = true
|
o.set[name] = true
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func validate(o *Opt, val string) error {
|
// parseValue converts a raw string to the typed value for o.Kind, validating it
|
||||||
|
// in the process: it is the single place a value is parsed. Bool/Int/Size/Float
|
||||||
|
// return their typed value (read back by the getters); Str/List/Enum need no
|
||||||
|
// stored conversion and return nil, with Enum still checked for membership.
|
||||||
|
func parseValue(o *Opt, val string) (any, error) {
|
||||||
switch o.Kind {
|
switch o.Kind {
|
||||||
case Bool:
|
case Bool:
|
||||||
// Booleans accept true/false plus the yes/no/1/0/on/off spellings, but
|
// Booleans accept exactly true/false (the boolWords vocabulary the Bool
|
||||||
// nothing else, so e.g. --enable-dht=flase is a parse error. The accepted
|
// getter honours); --enable-dht=flase is a parse error.
|
||||||
// set is boolWords, the same vocabulary the readers honour.
|
v, ok := boolWord(val)
|
||||||
if _, ok := boolWord(val); !ok {
|
if !ok {
|
||||||
return fmt.Errorf("--%s: %q is not a boolean (true/false)", o.Long, val)
|
return nil, fmt.Errorf("--%s: %q is not a boolean (true/false)", o.Long, val)
|
||||||
}
|
}
|
||||||
|
return v, nil
|
||||||
case Int:
|
case Int:
|
||||||
n, err := strconv.ParseInt(strings.TrimSpace(val), 10, 64)
|
n, err := strconv.ParseInt(strings.TrimSpace(val), 10, 64)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("--%s: %q is not an integer", o.Long, val)
|
return nil, fmt.Errorf("--%s: %q is not an integer", o.Long, val)
|
||||||
}
|
}
|
||||||
if err := checkBounds(o, n); err != nil {
|
if err := checkBounds(o, n); err != nil {
|
||||||
return err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
return n, nil
|
||||||
case Size:
|
case Size:
|
||||||
n, err := parseSize(val)
|
n, err := parseSize(val)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("--%s: %v", o.Long, err)
|
return nil, fmt.Errorf("--%s: %v", o.Long, err)
|
||||||
}
|
}
|
||||||
if err := checkBounds(o, n); err != nil {
|
if err := checkBounds(o, n); err != nil {
|
||||||
return err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
return n, nil
|
||||||
case Float:
|
case Float:
|
||||||
// seed-time and seed-ratio are both rates/durations that must be
|
// seed-time and seed-ratio must be non-negative: a negative SeedTime would
|
||||||
// non-negative; a negative SeedTime would fire the seed timer immediately
|
// fire the seed timer immediately and a negative SeedRatio is meaningless.
|
||||||
// and a negative SeedRatio is silently ignored. Reject negatives here.
|
|
||||||
// Opt.Min/Max are int64, so we only enforce the floor, not full bounds.
|
|
||||||
f, err := strconv.ParseFloat(strings.TrimSpace(val), 64)
|
f, err := strconv.ParseFloat(strings.TrimSpace(val), 64)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("--%s: %q is not a number", o.Long, val)
|
return nil, fmt.Errorf("--%s: %q is not a number", o.Long, val)
|
||||||
}
|
}
|
||||||
if f < 0 {
|
if f < 0 {
|
||||||
return fmt.Errorf("--%s: %v below minimum 0", o.Long, f)
|
return nil, fmt.Errorf("--%s: %v below minimum 0", o.Long, f)
|
||||||
}
|
}
|
||||||
|
return f, nil
|
||||||
case Enum:
|
case Enum:
|
||||||
for _, e := range o.Enum {
|
for _, e := range o.Enum {
|
||||||
if val == e {
|
if val == e {
|
||||||
return nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return fmt.Errorf("--%s: %q not one of %s", o.Long, val, strings.Join(o.Enum, "|"))
|
return nil, fmt.Errorf("--%s: %q not one of %s", o.Long, val, strings.Join(o.Enum, "|"))
|
||||||
}
|
}
|
||||||
return nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// checkBounds enforces an Int/Size option's numeric range. Min defaults to a
|
// checkBounds enforces an Int/Size option's numeric range. Min defaults to a
|
||||||
@@ -304,8 +324,3 @@ func confPath(explicit string) (path string, fromUser bool) {
|
|||||||
}
|
}
|
||||||
return filepath.Join(home, ".config", "got", "got.conf"), false
|
return filepath.Join(home, ".config", "got", "got.conf"), false
|
||||||
}
|
}
|
||||||
|
|
||||||
func truthy(s string) bool {
|
|
||||||
val, _ := boolWord(s)
|
|
||||||
return val
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ func TestParseArgs(t *testing.T) {
|
|||||||
wantVal map[string]string
|
wantVal map[string]string
|
||||||
uris []string
|
uris []string
|
||||||
}{
|
}{
|
||||||
{"short attached", []string{"-x16", "https://e.com"}, Run, map[string]string{"max-connection-per-server": "16"}, []string{"https://e.com"}},
|
{"short attached", []string{"-m3", "https://e.com"}, Run, map[string]string{"max-tries": "3"}, []string{"https://e.com"}},
|
||||||
{"long equals + bool", []string{"--split=8", "-c", "u"}, Run, map[string]string{"split": "8", "continue": "true"}, []string{"u"}},
|
{"long equals + bool", []string{"--split=8", "-c", "u"}, Run, map[string]string{"split": "8", "continue": "true"}, []string{"u"}},
|
||||||
{"long separate value", []string{"--out", "f.bin", "u"}, Run, map[string]string{"out": "f.bin"}, []string{"u"}},
|
{"long separate value", []string{"--out", "f.bin", "u"}, Run, map[string]string{"out": "f.bin"}, []string{"u"}},
|
||||||
{"bool bundle", []string{"-cq", "u"}, Run, map[string]string{"continue": "true", "quiet": "true"}, []string{"u"}},
|
{"bool bundle", []string{"-cq", "u"}, Run, map[string]string{"continue": "true", "quiet": "true"}, []string{"u"}},
|
||||||
@@ -22,7 +22,7 @@ func TestParseArgs(t *testing.T) {
|
|||||||
{"permute flag after uri", []string{"u", "--split=8"}, Run, map[string]string{"split": "8"}, []string{"u"}},
|
{"permute flag after uri", []string{"u", "--split=8"}, Run, map[string]string{"split": "8"}, []string{"u"}},
|
||||||
{"permute interleaved", []string{"a", "-c", "b", "--quiet"}, Run, map[string]string{"continue": "true", "quiet": "true"}, []string{"a", "b"}},
|
{"permute interleaved", []string{"a", "-c", "b", "--quiet"}, Run, map[string]string{"continue": "true", "quiet": "true"}, []string{"a", "b"}},
|
||||||
{"dash dash terminator", []string{"--split=8", "--", "--not-a-flag", "u"}, Run, map[string]string{"split": "8"}, []string{"--not-a-flag", "u"}},
|
{"dash dash terminator", []string{"--split=8", "--", "--not-a-flag", "u"}, Run, map[string]string{"split": "8"}, []string{"--not-a-flag", "u"}},
|
||||||
{"bt metadata timeout", []string{"--bt-metadata-timeout=120", "magnet:x"}, Run, map[string]string{"bt-metadata-timeout": "120"}, []string{"magnet:x"}},
|
{"long int + magnet uri", []string{"--timeout=120", "magnet:x"}, Run, map[string]string{"timeout": "120"}, []string{"magnet:x"}},
|
||||||
{"help", []string{"--help"}, ShowHelp, nil, nil},
|
{"help", []string{"--help"}, ShowHelp, nil, nil},
|
||||||
{"help short", []string{"-h"}, ShowHelp, nil, nil},
|
{"help short", []string{"-h"}, ShowHelp, nil, nil},
|
||||||
{"version", []string{"-v"}, ShowVersion, nil, nil},
|
{"version", []string{"-v"}, ShowVersion, nil, nil},
|
||||||
@@ -68,21 +68,6 @@ func TestParseArgsErrors(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// got bounds the magnet metadata fetch by default rather than waiting forever;
|
|
||||||
// guard the default value and its zero floor.
|
|
||||||
func TestBTMetadataTimeoutDefault(t *testing.T) {
|
|
||||||
o, ok := byLong["bt-metadata-timeout"]
|
|
||||||
if !ok {
|
|
||||||
t.Fatal("bt-metadata-timeout option is missing")
|
|
||||||
}
|
|
||||||
if o.Default != "60" {
|
|
||||||
t.Errorf("bt-metadata-timeout default = %q, want %q", o.Default, "60")
|
|
||||||
}
|
|
||||||
if o.Min != 0 {
|
|
||||||
t.Errorf("bt-metadata-timeout Min = %d, want 0 (reject negatives, allow 0=off)", o.Min)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseSize(t *testing.T) {
|
func TestParseSize(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
in string
|
in string
|
||||||
@@ -109,73 +94,84 @@ func TestParseSize(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestValidateRange(t *testing.T) {
|
func TestParseValueRange(t *testing.T) {
|
||||||
x := byLong["max-connection-per-server"]
|
// A synthetic bounded Int exercises the Min/Max range check independent of
|
||||||
if err := validate(x, "16"); err != nil {
|
// which options happen to carry bounds.
|
||||||
t.Errorf("validate 16: %v", err)
|
o := &Opt{Long: "n", Kind: Int, Min: 1, Max: 16}
|
||||||
|
if _, err := parseValue(o, "16"); err != nil {
|
||||||
|
t.Errorf("parseValue 16: %v", err)
|
||||||
}
|
}
|
||||||
if err := validate(x, "99"); err == nil {
|
if _, err := parseValue(o, "99"); err == nil {
|
||||||
t.Errorf("validate 99: want out-of-range error")
|
t.Errorf("parseValue 99: want out-of-range error")
|
||||||
|
}
|
||||||
|
if _, err := parseValue(o, "0"); err == nil {
|
||||||
|
t.Errorf("parseValue 0: want below-minimum error")
|
||||||
}
|
}
|
||||||
a := byLong["file-allocation"]
|
a := byLong["file-allocation"]
|
||||||
if err := validate(a, "bogus"); err == nil {
|
if _, err := parseValue(a, "bogus"); err == nil {
|
||||||
t.Errorf("validate enum bogus: want error")
|
t.Errorf("parseValue enum bogus: want error")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestValidateIntFloor(t *testing.T) {
|
func TestParseValueIntFloor(t *testing.T) {
|
||||||
// Min:0 now enforces a floor of 0: negatives are rejected.
|
// Min:0 enforces a floor of 0: negatives are rejected.
|
||||||
m := byLong["max-tries"] // Min 0, "0 = unlimited" runtime meaning preserved
|
m := byLong["max-tries"] // Min 0, "0 = unlimited" runtime meaning preserved
|
||||||
if err := validate(m, "0"); err != nil {
|
if _, err := parseValue(m, "0"); err != nil {
|
||||||
t.Errorf("validate max-tries 0: %v, want nil (0 stays a valid input)", err)
|
t.Errorf("parseValue max-tries 0: %v, want nil (0 stays a valid input)", err)
|
||||||
}
|
}
|
||||||
if err := validate(m, "-1"); err == nil {
|
if _, err := parseValue(m, "-1"); err == nil {
|
||||||
t.Errorf("validate max-tries -1: want below-minimum error")
|
t.Errorf("parseValue max-tries -1: want below-minimum error")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestValidateFloatNonNegative(t *testing.T) {
|
func TestParseValueFloatNonNegative(t *testing.T) {
|
||||||
// Float options (seed-time, seed-ratio) reject negatives: a negative
|
// Float options (seed-time, seed-ratio) reject negatives: a negative
|
||||||
// SeedTime would fire the seed timer immediately and a negative SeedRatio
|
// SeedTime would fire the seed timer immediately and a negative SeedRatio
|
||||||
// is silently ignored, so neither is a meaningful input.
|
// is silently ignored, so neither is a meaningful input.
|
||||||
for _, name := range []string{"seed-ratio", "seed-time"} {
|
for _, name := range []string{"seed-ratio", "seed-time"} {
|
||||||
o := byLong[name]
|
o := byLong[name]
|
||||||
if err := validate(o, "-1"); err == nil {
|
if _, err := parseValue(o, "-1"); err == nil {
|
||||||
t.Errorf("validate %s -1: want below-minimum error", name)
|
t.Errorf("parseValue %s -1: want below-minimum error", name)
|
||||||
}
|
}
|
||||||
if err := validate(o, "0"); err != nil {
|
if _, err := parseValue(o, "0"); err != nil {
|
||||||
t.Errorf("validate %s 0: %v, want nil", name, err)
|
t.Errorf("parseValue %s 0: %v, want nil", name, err)
|
||||||
}
|
}
|
||||||
if err := validate(o, "1.5"); err != nil {
|
if _, err := parseValue(o, "1.5"); err != nil {
|
||||||
t.Errorf("validate %s 1.5: %v, want nil", name, err)
|
t.Errorf("parseValue %s 1.5: %v, want nil", name, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestValidateBool(t *testing.T) {
|
func TestParseValueBool(t *testing.T) {
|
||||||
b := byLong["enable-dht"]
|
b := byLong["enable-dht"]
|
||||||
for _, ok := range []string{"true", "false", "yes", "no", "1", "0", "on", "off"} {
|
// Only literal true/false are accepted (case-sensitive), with surrounding
|
||||||
if err := validate(b, ok); err != nil {
|
// whitespace trimmed.
|
||||||
t.Errorf("validate bool %q: %v", ok, err)
|
for _, ok := range []string{"true", "false", " true ", "false\t"} {
|
||||||
|
if _, err := parseValue(b, ok); err != nil {
|
||||||
|
t.Errorf("parseValue bool %q: %v", ok, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if err := validate(b, "flase"); err == nil {
|
// The invented spellings and any case variant must fail.
|
||||||
t.Errorf("validate bool flase: want error")
|
for _, bad := range []string{"yes", "no", "1", "0", "on", "off", "flase",
|
||||||
|
"True", "TRUE", "tRuE", "False", "FALSE"} {
|
||||||
|
if _, err := parseValue(b, bad); err == nil {
|
||||||
|
t.Errorf("parseValue bool %q: want error (only true/false accepted)", bad)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestValidateSizeBounds(t *testing.T) {
|
func TestParseValueSizeBounds(t *testing.T) {
|
||||||
k := byLong["min-split-size"] // 1M..1024M (no G unit)
|
k := &Opt{Long: "sz", Kind: Size, Min: 1 << 20, Max: 1 << 30} // 1M..1024M
|
||||||
if err := validate(k, "20M"); err != nil {
|
if _, err := parseValue(k, "20M"); err != nil {
|
||||||
t.Errorf("validate min-split-size 20M: %v", err)
|
t.Errorf("parseValue 20M: %v", err)
|
||||||
}
|
}
|
||||||
if err := validate(k, "512K"); err == nil {
|
if _, err := parseValue(k, "512K"); err == nil {
|
||||||
t.Errorf("validate min-split-size 512K: want below-minimum error")
|
t.Errorf("parseValue 512K: want below-minimum error")
|
||||||
}
|
}
|
||||||
// 1025M exceeds the 1<<30 cap and still parses, so it tests the bounds path
|
// 1025M exceeds the 1<<30 cap and still parses, so it tests the bounds path
|
||||||
// (where "2G" would have failed earlier as an unknown unit).
|
// (where "2G" would have failed earlier as an unknown unit).
|
||||||
if err := validate(k, "1025M"); err == nil {
|
if _, err := parseValue(k, "1025M"); err == nil {
|
||||||
t.Errorf("validate min-split-size 1025M: want above-maximum error")
|
t.Errorf("parseValue 1025M: want above-maximum error")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -213,14 +209,17 @@ func TestExplicitConfMissingFatal(t *testing.T) {
|
|||||||
func TestOptionsGetters(t *testing.T) {
|
func TestOptionsGetters(t *testing.T) {
|
||||||
o := newOptions()
|
o := newOptions()
|
||||||
o.vals["split"] = "8"
|
o.vals["split"] = "8"
|
||||||
o.vals["min-split-size"] = "20M"
|
o.vals["max-overall-download-limit"] = "20M"
|
||||||
o.vals["continue"] = "true"
|
o.vals["continue"] = "true"
|
||||||
o.vals["seed-ratio"] = "1.5"
|
o.vals["seed-ratio"] = "1.5"
|
||||||
|
if err := o.finalize(); err != nil { // parse the raw values once, as Parse does
|
||||||
|
t.Fatalf("finalize: %v", err)
|
||||||
|
}
|
||||||
if o.Int("split") != 8 {
|
if o.Int("split") != 8 {
|
||||||
t.Errorf("Int split = %d", o.Int("split"))
|
t.Errorf("Int split = %d", o.Int("split"))
|
||||||
}
|
}
|
||||||
if o.Size("min-split-size") != 20<<20 {
|
if o.Size("max-overall-download-limit") != 20<<20 {
|
||||||
t.Errorf("Size min-split-size = %d", o.Size("min-split-size"))
|
t.Errorf("Size max-overall-download-limit = %d", o.Size("max-overall-download-limit"))
|
||||||
}
|
}
|
||||||
if !o.Bool("continue") {
|
if !o.Bool("continue") {
|
||||||
t.Errorf("Bool continue = false")
|
t.Errorf("Bool continue = false")
|
||||||
|
|||||||
@@ -55,9 +55,6 @@ type Stat struct {
|
|||||||
Seeders int // connected seeders (BitTorrent only)
|
Seeders int // connected seeders (BitTorrent only)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Done reports whether the download has reached a terminal state.
|
|
||||||
func (s Stat) Done() bool { return s.Status == Complete || s.Status == Errored }
|
|
||||||
|
|
||||||
// A Download is one logical job: a URL, a torrent, a magnet. Run blocks in its
|
// A Download is one logical job: a URL, a torrent, a magnet. Run blocks in its
|
||||||
// own goroutine until the work finishes, fails, or ctx is cancelled. Stat may
|
// own goroutine until the work finishes, fails, or ctx is cancelled. Stat may
|
||||||
// be called concurrently at any time and must not block; implementations back
|
// be called concurrently at any time and must not block; implementations back
|
||||||
@@ -67,3 +64,14 @@ type Download interface {
|
|||||||
Run(ctx context.Context) error
|
Run(ctx context.Context) error
|
||||||
Stat() Stat
|
Stat() Stat
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Seeder is an optional capability of a Download whose work continues after its
|
||||||
|
// content is fully downloaded — a torrent that keeps running to seed.
|
||||||
|
// SeedingStarted is closed at that download->seed transition. The engine uses it
|
||||||
|
// to free the concurrency slot the moment a download stops downloading: a
|
||||||
|
// seed-only torrent is no longer a download, so it must not count against
|
||||||
|
// --max-concurrent-downloads (-j) and block a queued download. HTTP downloads do
|
||||||
|
// not implement this, so they hold their slot until they finish.
|
||||||
|
type Seeder interface {
|
||||||
|
SeedingStarted() <-chan struct{}
|
||||||
|
}
|
||||||
|
|||||||
@@ -46,6 +46,12 @@ func (e *Engine) Run(ctx context.Context, downloads []Download) []Result {
|
|||||||
go func(i int, d Download) {
|
go func(i int, d Download) {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
|
|
||||||
|
// Track immediately so a download still queued for a concurrency slot
|
||||||
|
// is visible to the progress reporter as Waiting, rather than hidden
|
||||||
|
// until it starts running.
|
||||||
|
e.track(d)
|
||||||
|
defer e.untrack(d)
|
||||||
|
|
||||||
// Acquire a concurrency slot, or bail if we're shutting down
|
// Acquire a concurrency slot, or bail if we're shutting down
|
||||||
// before this download ever started.
|
// before this download ever started.
|
||||||
select {
|
select {
|
||||||
@@ -54,10 +60,24 @@ func (e *Engine) Run(ctx context.Context, downloads []Download) []Result {
|
|||||||
results <- Result{Name: d.Name(), Index: i, Err: ctx.Err(), Stat: d.Stat()}
|
results <- Result{Name: d.Name(), Index: i, Err: ctx.Err(), Stat: d.Stat()}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
defer func() { <-slots }()
|
// Release the slot exactly once — whether the download finishes or, for
|
||||||
|
// a torrent, first detaches into seed-only. A seeding torrent is no
|
||||||
e.track(d)
|
// longer downloading, so it stops counting against -j and lets a queued
|
||||||
defer e.untrack(d)
|
// download start; this goroutine lives on to seed.
|
||||||
|
var once sync.Once
|
||||||
|
release := func() { once.Do(func() { <-slots }) }
|
||||||
|
defer release()
|
||||||
|
if s, ok := d.(Seeder); ok {
|
||||||
|
done := make(chan struct{})
|
||||||
|
defer close(done)
|
||||||
|
go func() {
|
||||||
|
select {
|
||||||
|
case <-s.SeedingStarted():
|
||||||
|
release()
|
||||||
|
case <-done:
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
err := d.Run(ctx)
|
err := d.Run(ctx)
|
||||||
results <- Result{Name: d.Name(), Index: i, Err: err, Stat: d.Stat()}
|
results <- Result{Name: d.Name(), Index: i, Err: err, Stat: d.Stat()}
|
||||||
@@ -74,7 +94,8 @@ func (e *Engine) Run(ctx context.Context, downloads []Download) []Result {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// Snapshot returns a Stat for every currently running download.
|
// Snapshot returns a Stat for every tracked download — those running plus those
|
||||||
|
// still queued for a concurrency slot (reported as Waiting).
|
||||||
func (e *Engine) Snapshot() []Stat {
|
func (e *Engine) Snapshot() []Stat {
|
||||||
e.mu.Lock()
|
e.mu.Lock()
|
||||||
defer e.mu.Unlock()
|
defer e.mu.Unlock()
|
||||||
@@ -91,6 +112,10 @@ func (e *Engine) track(d Download) {
|
|||||||
e.mu.Unlock()
|
e.mu.Unlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// untrack removes d from the active set. Every Download implementation is a
|
||||||
|
// pointer type, so the == compares identities — never the fields of a value,
|
||||||
|
// which could panic on a non-comparable one. (Keying on Stat().ID is not an
|
||||||
|
// option: a torrent's ID changes from source to infohash once metadata loads.)
|
||||||
func (e *Engine) untrack(d Download) {
|
func (e *Engine) untrack(d Download) {
|
||||||
e.mu.Lock()
|
e.mu.Lock()
|
||||||
defer e.mu.Unlock()
|
defer e.mu.Unlock()
|
||||||
|
|||||||
107
download/engine_test.go
Normal file
107
download/engine_test.go
Normal file
@@ -0,0 +1,107 @@
|
|||||||
|
package download
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// fakeDownload is a Download that blocks in Run until released, and (optionally)
|
||||||
|
// announces a download->seed transition so the engine can free its -j slot.
|
||||||
|
type fakeDownload struct {
|
||||||
|
name string
|
||||||
|
started chan struct{} // closed when Run begins
|
||||||
|
seeding chan struct{} // SeedingStarted signal
|
||||||
|
release chan struct{} // Run returns once this is closed
|
||||||
|
detach bool // if set, Run signals seeding right after starting
|
||||||
|
}
|
||||||
|
|
||||||
|
func newFake(name string, detach bool) *fakeDownload {
|
||||||
|
return &fakeDownload{
|
||||||
|
name: name,
|
||||||
|
started: make(chan struct{}),
|
||||||
|
seeding: make(chan struct{}),
|
||||||
|
release: make(chan struct{}),
|
||||||
|
detach: detach,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeDownload) Name() string { return f.name }
|
||||||
|
func (f *fakeDownload) Stat() Stat { return Stat{Name: f.name} }
|
||||||
|
func (f *fakeDownload) SeedingStarted() <-chan struct{} { return f.seeding }
|
||||||
|
|
||||||
|
func (f *fakeDownload) Run(ctx context.Context) error {
|
||||||
|
close(f.started)
|
||||||
|
if f.detach {
|
||||||
|
close(f.seeding)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-f.release:
|
||||||
|
return nil
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func started(f *fakeDownload, d time.Duration) bool {
|
||||||
|
select {
|
||||||
|
case <-f.started:
|
||||||
|
return true
|
||||||
|
case <-time.After(d):
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A torrent that has finished downloading and is now seeding must not keep
|
||||||
|
// occupying a -j slot: with -j 1, a seeding download has to let a queued one run.
|
||||||
|
func TestEngineSeedingFreesSlot(t *testing.T) {
|
||||||
|
eng := NewEngine(1)
|
||||||
|
a := newFake("a", true)
|
||||||
|
b := newFake("b", true)
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
done := make(chan []Result, 1)
|
||||||
|
go func() { done <- eng.Run(ctx, []Download{a, b}) }()
|
||||||
|
|
||||||
|
// Neither is ever released, yet both must start: each frees the single slot
|
||||||
|
// the instant it detaches into seeding.
|
||||||
|
if !started(a, time.Second) || !started(b, time.Second) {
|
||||||
|
t.Fatal("both downloads should start once seeding frees the -j slot")
|
||||||
|
}
|
||||||
|
close(a.release)
|
||||||
|
close(b.release)
|
||||||
|
<-done
|
||||||
|
}
|
||||||
|
|
||||||
|
// Control: without a seeding transition, -j 1 still serializes — the second
|
||||||
|
// download waits for the first to finish before it starts.
|
||||||
|
func TestEngineLimitsConcurrentDownloads(t *testing.T) {
|
||||||
|
eng := NewEngine(1)
|
||||||
|
a := newFake("a", false)
|
||||||
|
b := newFake("b", false)
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
done := make(chan []Result, 1)
|
||||||
|
go func() { done <- eng.Run(ctx, []Download{a, b}) }()
|
||||||
|
|
||||||
|
var first, second *fakeDownload
|
||||||
|
select {
|
||||||
|
case <-a.started:
|
||||||
|
first, second = a, b
|
||||||
|
case <-b.started:
|
||||||
|
first, second = b, a
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("one download should start")
|
||||||
|
}
|
||||||
|
if started(second, 100*time.Millisecond) {
|
||||||
|
t.Fatal("second download must not start while the only slot is held")
|
||||||
|
}
|
||||||
|
close(first.release) // first finishes, freeing the slot
|
||||||
|
if !started(second, time.Second) {
|
||||||
|
t.Fatal("second download should start after the first frees the slot")
|
||||||
|
}
|
||||||
|
close(second.release)
|
||||||
|
<-done
|
||||||
|
}
|
||||||
4
go.mod
4
go.mod
@@ -3,7 +3,7 @@ module github.com/hanbok/got
|
|||||||
go 1.25
|
go 1.25
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/anacrolix/missinggo/v2 v2.10.0
|
github.com/anacrolix/log v0.17.1-0.20251118025802-918f1157b7bb
|
||||||
github.com/anacrolix/torrent v1.61.0
|
github.com/anacrolix/torrent v1.61.0
|
||||||
golang.org/x/sys v0.38.0
|
golang.org/x/sys v0.38.0
|
||||||
golang.org/x/term v0.37.0
|
golang.org/x/term v0.37.0
|
||||||
@@ -19,9 +19,9 @@ require (
|
|||||||
github.com/anacrolix/envpprof v1.4.0 // indirect
|
github.com/anacrolix/envpprof v1.4.0 // indirect
|
||||||
github.com/anacrolix/generics v0.1.1-0.20251125230353-15d98d46693b // indirect
|
github.com/anacrolix/generics v0.1.1-0.20251125230353-15d98d46693b // indirect
|
||||||
github.com/anacrolix/go-libutp v1.3.2 // indirect
|
github.com/anacrolix/go-libutp v1.3.2 // indirect
|
||||||
github.com/anacrolix/log v0.17.1-0.20251118025802-918f1157b7bb // indirect
|
|
||||||
github.com/anacrolix/missinggo v1.3.0 // indirect
|
github.com/anacrolix/missinggo v1.3.0 // indirect
|
||||||
github.com/anacrolix/missinggo/perf v1.0.0 // indirect
|
github.com/anacrolix/missinggo/perf v1.0.0 // indirect
|
||||||
|
github.com/anacrolix/missinggo/v2 v2.10.0 // indirect
|
||||||
github.com/anacrolix/mmsg v1.0.1 // indirect
|
github.com/anacrolix/mmsg v1.0.1 // indirect
|
||||||
github.com/anacrolix/multiless v0.4.0 // indirect
|
github.com/anacrolix/multiless v0.4.0 // indirect
|
||||||
github.com/anacrolix/stm v0.5.0 // indirect
|
github.com/anacrolix/stm v0.5.0 // indirect
|
||||||
|
|||||||
@@ -1,22 +0,0 @@
|
|||||||
package httpdl
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestLoadCAPool(t *testing.T) {
|
|
||||||
// A missing file is an error, not a silent fall-back to the system roots.
|
|
||||||
if _, err := loadCAPool(filepath.Join(t.TempDir(), "absent.pem")); err == nil {
|
|
||||||
t.Errorf("loadCAPool(missing): want error")
|
|
||||||
}
|
|
||||||
// A file with no PEM certificates is an error (AppendCertsFromPEM found none).
|
|
||||||
bad := filepath.Join(t.TempDir(), "bad.pem")
|
|
||||||
if err := os.WriteFile(bad, []byte("not a certificate\n"), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := loadCAPool(bad); err == nil {
|
|
||||||
t.Errorf("loadCAPool(garbage): want error (no certificates)")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,97 +0,0 @@
|
|||||||
package httpdl
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/hanbok/got/download"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestConditionalGet304 checks that --conditional-get sends If-Modified-Since
|
|
||||||
// for an existing local file and treats a 304 as success without rewriting the
|
|
||||||
// file.
|
|
||||||
func TestConditionalGet304(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
out := filepath.Join(dir, "file.bin")
|
|
||||||
const existing = "already here"
|
|
||||||
if err := os.WriteFile(out, []byte(existing), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
gotIfModSince := false
|
|
||||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.Header.Get("If-Modified-Since") != "" {
|
|
||||||
gotIfModSince = true
|
|
||||||
}
|
|
||||||
w.WriteHeader(http.StatusNotModified)
|
|
||||||
}))
|
|
||||||
defer srv.Close()
|
|
||||||
|
|
||||||
d := New([]string{srv.URL + "/file.bin"}, Config{
|
|
||||||
Dir: dir,
|
|
||||||
Out: "file.bin",
|
|
||||||
MaxConnPerServer: 1,
|
|
||||||
ConditionalGet: true,
|
|
||||||
AllowOverwrite: true,
|
|
||||||
UserAgent: "got-test",
|
|
||||||
})
|
|
||||||
if err := d.Run(context.Background()); err != nil {
|
|
||||||
t.Fatalf("conditional-get 304 should succeed, got %v", err)
|
|
||||||
}
|
|
||||||
if !gotIfModSince {
|
|
||||||
t.Fatal("expected an If-Modified-Since header on the request")
|
|
||||||
}
|
|
||||||
if d.Stat().Status != download.Complete {
|
|
||||||
t.Fatalf("status = %v, want Complete", d.Stat().Status)
|
|
||||||
}
|
|
||||||
// The file must be untouched: 304 means "up to date".
|
|
||||||
b, err := os.ReadFile(out)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if string(b) != existing {
|
|
||||||
t.Fatalf("file was rewritten: %q, want %q", b, existing)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestConditionalGetSkippedWithControlFile verifies the rule that
|
|
||||||
// --conditional-get is ignored when a .got control file exists, so an
|
|
||||||
// in-progress resume is never short-circuited by a 304.
|
|
||||||
func TestConditionalGetSkippedWithControlFile(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
out := filepath.Join(dir, "file.bin")
|
|
||||||
if err := os.WriteFile(out, []byte("partial"), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
// A control file present means a resume is in flight.
|
|
||||||
if err := os.WriteFile(controlPath(out), []byte("{}"), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
sawIfModSince := false
|
|
||||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.Header.Get("If-Modified-Since") != "" {
|
|
||||||
sawIfModSince = true
|
|
||||||
}
|
|
||||||
// Serve a small body so the run can complete.
|
|
||||||
http.Error(w, "no range", http.StatusOK)
|
|
||||||
}))
|
|
||||||
defer srv.Close()
|
|
||||||
|
|
||||||
d := New([]string{srv.URL + "/file.bin"}, Config{
|
|
||||||
Dir: dir,
|
|
||||||
Out: "file.bin",
|
|
||||||
MaxConnPerServer: 1,
|
|
||||||
ConditionalGet: true,
|
|
||||||
AllowOverwrite: true,
|
|
||||||
UserAgent: "got-test",
|
|
||||||
})
|
|
||||||
_ = d.Run(context.Background())
|
|
||||||
if sawIfModSince {
|
|
||||||
t.Fatal("If-Modified-Since must not be sent when a control file exists")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -27,7 +27,9 @@ type segState struct {
|
|||||||
|
|
||||||
func controlPath(out string) string { return out + ".got" }
|
func controlPath(out string) string { return out + ".got" }
|
||||||
|
|
||||||
// snapshot builds a control record from the live segments.
|
// snapshot builds a control record from the live segments. The segment slice is
|
||||||
|
// fixed once the file is divided and each segment's frontier is owned by one
|
||||||
|
// worker, so reading written atomically gives a consistent record with no lock.
|
||||||
func snapshot(url string, total int64, etag, lastmod string, segs []seg) control {
|
func snapshot(url string, total int64, etag, lastmod string, segs []seg) control {
|
||||||
c := control{URL: url, Total: total, ETag: etag, LastModified: lastmod, Segs: make([]segState, len(segs))}
|
c := control{URL: url, Total: total, ETag: etag, LastModified: lastmod, Segs: make([]segState, len(segs))}
|
||||||
for i := range segs {
|
for i := range segs {
|
||||||
@@ -73,7 +75,34 @@ func loadControl(out, url string, total int64, etag, lastmod string) *control {
|
|||||||
if (lastmod != "" || c.LastModified != "") && c.LastModified != lastmod {
|
if (lastmod != "" || c.LastModified != "") && c.LastModified != lastmod {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
// Reject a control file whose segments do not exactly tile [0,total): a
|
||||||
|
// truncated/corrupted/hand-edited sidecar that still parses as JSON could
|
||||||
|
// otherwise mark a segment done() without its bytes on disk (inflated Written)
|
||||||
|
// or leave an un-downloaded hole, both of which would be reported as a complete
|
||||||
|
// file. We restart cleanly instead of trusting it.
|
||||||
|
if !validSegs(c.Segs, c.Total) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
return &c
|
return &c
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// validSegs reports whether segs cover [0,total) with no gap or overlap and a
|
||||||
|
// sane written count for each. got always writes segments in ascending start
|
||||||
|
// order, so coverage is checked in place; an out-of-order sidecar (only possible
|
||||||
|
// from a hand-edited or corrupted file) is rejected, which restarts cleanly.
|
||||||
|
func validSegs(segs []segState, total int64) bool {
|
||||||
|
if total <= 0 || len(segs) == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
var next int64
|
||||||
|
for _, s := range segs {
|
||||||
|
length := s.End - s.Start + 1
|
||||||
|
if s.Start != next || s.End < s.Start || s.Written < 0 || s.Written > length {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
next = s.End + 1
|
||||||
|
}
|
||||||
|
return next == total
|
||||||
|
}
|
||||||
|
|
||||||
func removeControl(out string) { os.Remove(controlPath(out)) }
|
func removeControl(out string) { os.Remove(controlPath(out)) }
|
||||||
|
|||||||
@@ -1,177 +0,0 @@
|
|||||||
package httpdl
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bufio"
|
|
||||||
"fmt"
|
|
||||||
"math"
|
|
||||||
"net/http"
|
|
||||||
"net/http/cookiejar"
|
|
||||||
"net/url"
|
|
||||||
"os"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Cookie handling implements the --load-cookies / --save-cookies options, which
|
|
||||||
// read and write the Netscape/Mozilla cookies.txt format (one tab-separated
|
|
||||||
// cookie per line). We back it with the standard net/http/cookiejar so the
|
|
||||||
// http.Client applies and tracks Set-Cookie headers automatically; the file
|
|
||||||
// format is just the on-disk representation of that jar.
|
|
||||||
|
|
||||||
// loadCookieJar builds a cookie jar seeded from a Netscape cookies.txt file. A
|
|
||||||
// missing file yields an empty jar (not an error): there are simply no cookies
|
|
||||||
// to load. A malformed line is skipped, giving per-line tolerance.
|
|
||||||
func loadCookieJar(path string) (*cookiejar.Jar, error) {
|
|
||||||
jar, err := cookiejar.New(nil)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if path == "" {
|
|
||||||
return jar, nil
|
|
||||||
}
|
|
||||||
f, err := os.Open(path)
|
|
||||||
if err != nil {
|
|
||||||
if os.IsNotExist(err) {
|
|
||||||
return jar, nil // nothing to load yet
|
|
||||||
}
|
|
||||||
return nil, fmt.Errorf("load-cookies %s: %w", path, err)
|
|
||||||
}
|
|
||||||
defer f.Close()
|
|
||||||
|
|
||||||
// Group cookies by the URL their domain implies, since SetCookies keys on a
|
|
||||||
// URL. We synthesise a representative URL per (domain, secure, path).
|
|
||||||
byURL := map[string][]*http.Cookie{}
|
|
||||||
urls := map[string]*url.URL{}
|
|
||||||
sc := bufio.NewScanner(f)
|
|
||||||
for sc.Scan() {
|
|
||||||
line := sc.Text()
|
|
||||||
c, u := parseNetscapeCookie(line)
|
|
||||||
if c == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
key := u.String()
|
|
||||||
if _, ok := urls[key]; !ok {
|
|
||||||
urls[key] = u
|
|
||||||
}
|
|
||||||
byURL[key] = append(byURL[key], c)
|
|
||||||
}
|
|
||||||
if err := sc.Err(); err != nil {
|
|
||||||
return nil, fmt.Errorf("load-cookies %s: %w", path, err)
|
|
||||||
}
|
|
||||||
for key, cs := range byURL {
|
|
||||||
jar.SetCookies(urls[key], cs)
|
|
||||||
}
|
|
||||||
return jar, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseNetscapeCookie parses one cookies.txt line into a cookie plus the URL it
|
|
||||||
// belongs to, or (nil, nil) for a comment, blank or malformed line. The format
|
|
||||||
// is seven tab-separated fields:
|
|
||||||
//
|
|
||||||
// domain includeSubdomains path secure expiry name value
|
|
||||||
//
|
|
||||||
// Curl marks HttpOnly cookies with a "#HttpOnly_" line prefix rather than a
|
|
||||||
// real comment, so we recognise that prefix instead of dropping the line.
|
|
||||||
// Honouring HttpOnly_ rather than skipping every "#" line never loses a cookie.
|
|
||||||
func parseNetscapeCookie(line string) (*http.Cookie, *url.URL) {
|
|
||||||
httpOnly := false
|
|
||||||
if strings.HasPrefix(line, "#HttpOnly_") {
|
|
||||||
httpOnly = true
|
|
||||||
line = strings.TrimPrefix(line, "#HttpOnly_")
|
|
||||||
} else if strings.HasPrefix(line, "#") {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(line) == "" {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
f := strings.Split(line, "\t")
|
|
||||||
if len(f) < 6 {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
domain := f[0]
|
|
||||||
hostOnly := !strings.EqualFold(f[1], "TRUE")
|
|
||||||
path := f[2]
|
|
||||||
secure := strings.EqualFold(f[3], "TRUE")
|
|
||||||
name := f[5]
|
|
||||||
value := ""
|
|
||||||
if len(f) >= 7 {
|
|
||||||
value = f[6]
|
|
||||||
}
|
|
||||||
if name == "" || domain == "" {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
c := &http.Cookie{
|
|
||||||
Name: name,
|
|
||||||
Value: value,
|
|
||||||
Path: path,
|
|
||||||
Secure: secure,
|
|
||||||
HttpOnly: httpOnly,
|
|
||||||
}
|
|
||||||
// A non-host-only cookie (includeSubdomains TRUE) carries a Domain attribute
|
|
||||||
// so the jar applies it to subdomains; a host-only one leaves Domain empty so
|
|
||||||
// the jar binds it to the exact request host.
|
|
||||||
if !hostOnly {
|
|
||||||
c.Domain = domain
|
|
||||||
}
|
|
||||||
// expiry 0 means a session cookie; otherwise it is a Unix timestamp. The jar
|
|
||||||
// drops already-expired cookies on load.
|
|
||||||
if exp, err := strconv.ParseInt(strings.TrimSpace(f[4]), 10, 64); err == nil && exp > 0 {
|
|
||||||
c.Expires = time.Unix(exp, 0)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The jar needs a URL whose host and scheme match the cookie. A leading dot
|
|
||||||
// denotes a domain cookie; strip it for the host but keep the cookie Domain
|
|
||||||
// so the jar treats it as domain-wide.
|
|
||||||
host := strings.TrimPrefix(domain, ".")
|
|
||||||
scheme := "http"
|
|
||||||
if secure {
|
|
||||||
scheme = "https"
|
|
||||||
}
|
|
||||||
u := &url.URL{Scheme: scheme, Host: host, Path: path}
|
|
||||||
return c, u
|
|
||||||
}
|
|
||||||
|
|
||||||
// saveCookieJar writes every cookie the jar holds (for the hosts we have
|
|
||||||
// contacted) back to path in Netscape format, implementing --save-cookies on
|
|
||||||
// exit. Because net/http/cookiejar exposes cookies only per URL, we ask it for
|
|
||||||
// the cookies of each host we recorded a request against.
|
|
||||||
func saveCookieJar(jar *cookiejar.Jar, urls []*url.URL, path string) error {
|
|
||||||
if path == "" || jar == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
var b strings.Builder
|
|
||||||
b.WriteString("# Netscape HTTP Cookie File\n")
|
|
||||||
seen := map[string]bool{}
|
|
||||||
for _, u := range urls {
|
|
||||||
for _, c := range jar.Cookies(u) {
|
|
||||||
// Cookies() drops the domain/path/expiry, so reconstruct sensible
|
|
||||||
// values: host-only domain, root path, far-future expiry. This is the
|
|
||||||
// best a host-only readback can do; it round-trips the name/value pair
|
|
||||||
// that authenticated downloads actually need.
|
|
||||||
key := u.Host + "\x00" + c.Name
|
|
||||||
if seen[key] {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
seen[key] = true
|
|
||||||
expiry := int64(math.MaxInt32) // session-stable far future
|
|
||||||
line := strings.Join([]string{
|
|
||||||
u.Host, "FALSE", "/",
|
|
||||||
boolToTRUE(u.Scheme == "https"),
|
|
||||||
strconv.FormatInt(expiry, 10),
|
|
||||||
c.Name, c.Value,
|
|
||||||
}, "\t")
|
|
||||||
b.WriteString(line)
|
|
||||||
b.WriteByte('\n')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return os.WriteFile(path, []byte(b.String()), 0o600)
|
|
||||||
}
|
|
||||||
|
|
||||||
func boolToTRUE(b bool) string {
|
|
||||||
if b {
|
|
||||||
return "TRUE"
|
|
||||||
}
|
|
||||||
return "FALSE"
|
|
||||||
}
|
|
||||||
@@ -1,126 +0,0 @@
|
|||||||
package httpdl
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strconv"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestParseNetscapeCookie(t *testing.T) {
|
|
||||||
future := time.Now().Add(24 * time.Hour).Unix()
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
line string
|
|
||||||
wantNil bool
|
|
||||||
wantName string
|
|
||||||
wantValue string
|
|
||||||
wantDom string // expected http.Cookie.Domain (empty for host-only)
|
|
||||||
wantHTTP bool
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "domain cookie",
|
|
||||||
line: ".example.com\tTRUE\t/\tFALSE\t" + itoa(future) + "\tSID\tabc123",
|
|
||||||
wantName: "SID",
|
|
||||||
wantValue: "abc123",
|
|
||||||
wantDom: ".example.com",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "host-only cookie",
|
|
||||||
line: "example.com\tFALSE\t/\tTRUE\t" + itoa(future) + "\ttoken\txyz",
|
|
||||||
wantName: "token",
|
|
||||||
wantValue: "xyz",
|
|
||||||
wantDom: "", // host-only: no Domain attribute
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "httponly prefix kept",
|
|
||||||
line: "#HttpOnly_example.com\tFALSE\t/\tFALSE\t" + itoa(future) + "\tHO\tsecret",
|
|
||||||
wantName: "HO",
|
|
||||||
wantValue: "secret",
|
|
||||||
wantHTTP: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "empty value field",
|
|
||||||
line: "example.com\tFALSE\t/\tFALSE\t0\tflag\t",
|
|
||||||
wantName: "flag",
|
|
||||||
wantValue: "",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "comment line",
|
|
||||||
line: "# this is a comment",
|
|
||||||
wantNil: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "blank line",
|
|
||||||
line: " ",
|
|
||||||
wantNil: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "too few fields",
|
|
||||||
line: "example.com\tFALSE\t/\tFALSE\t0",
|
|
||||||
wantNil: true,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for _, tc := range tests {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
c, u := parseNetscapeCookie(tc.line)
|
|
||||||
if tc.wantNil {
|
|
||||||
if c != nil {
|
|
||||||
t.Fatalf("expected nil cookie, got %+v", c)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if c == nil {
|
|
||||||
t.Fatal("expected a cookie, got nil")
|
|
||||||
}
|
|
||||||
if c.Name != tc.wantName || c.Value != tc.wantValue {
|
|
||||||
t.Fatalf("name/value = %q/%q, want %q/%q", c.Name, c.Value, tc.wantName, tc.wantValue)
|
|
||||||
}
|
|
||||||
if c.Domain != tc.wantDom {
|
|
||||||
t.Fatalf("domain = %q, want %q", c.Domain, tc.wantDom)
|
|
||||||
}
|
|
||||||
if c.HttpOnly != tc.wantHTTP {
|
|
||||||
t.Fatalf("httponly = %v, want %v", c.HttpOnly, tc.wantHTTP)
|
|
||||||
}
|
|
||||||
if u == nil {
|
|
||||||
t.Fatal("expected a URL, got nil")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func itoa(n int64) string { return strconv.FormatInt(n, 10) }
|
|
||||||
func itoa64(n int64) string { return strconv.FormatInt(n, 10) }
|
|
||||||
|
|
||||||
func TestLoadCookieJarMissingFile(t *testing.T) {
|
|
||||||
// A missing cookies file is not an error: there are simply no cookies to load.
|
|
||||||
jar, err := loadCookieJar(filepath.Join(t.TempDir(), "nope.txt"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("missing file should not error: %v", err)
|
|
||||||
}
|
|
||||||
if jar == nil {
|
|
||||||
t.Fatal("expected an empty jar, got nil")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLoadCookieJarAppliesCookie(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
path := filepath.Join(dir, "cookies.txt")
|
|
||||||
future := time.Now().Add(24 * time.Hour).Unix()
|
|
||||||
data := "# Netscape HTTP Cookie File\n" +
|
|
||||||
".example.com\tTRUE\t/\tFALSE\t" + itoa64(future) + "\tSID\tabc123\n"
|
|
||||||
if err := os.WriteFile(path, []byte(data), 0o600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
jar, err := loadCookieJar(path)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
req, _ := http.NewRequest(http.MethodGet, "http://www.example.com/file", nil)
|
|
||||||
got := jar.Cookies(req.URL)
|
|
||||||
if len(got) != 1 || got[0].Name != "SID" || got[0].Value != "abc123" {
|
|
||||||
t.Fatalf("jar did not apply the domain cookie to a subdomain: %+v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
638
httpdl/httpdl.go
638
httpdl/httpdl.go
@@ -1,6 +1,6 @@
|
|||||||
// Package httpdl downloads a single HTTP(S) resource over one or more
|
// Package httpdl downloads a single HTTP(S) resource over one or more
|
||||||
// connections. The model is deliberately flat: split the file into byte-range
|
// connections. The model is deliberately flat: split the file into byte-range
|
||||||
// segments, hand them to a pool of worker goroutines, and have each worker
|
// segments, give each segment its own worker goroutine, and have each worker
|
||||||
// stream its range straight to disk with WriteAt (safe for concurrent,
|
// stream its range straight to disk with WriteAt (safe for concurrent,
|
||||||
// non-overlapping writes — no shared seek, no mutex). Goroutines blocking on
|
// non-overlapping writes — no shared seek, no mutex). Goroutines blocking on
|
||||||
// real I/O keep the model simple: no segment manager, no piece storage, no
|
// real I/O keep the model simple: no segment manager, no piece storage, no
|
||||||
@@ -9,8 +9,6 @@ package httpdl
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/tls"
|
|
||||||
"crypto/x509"
|
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"hash"
|
"hash"
|
||||||
@@ -18,7 +16,6 @@ import (
|
|||||||
"mime"
|
"mime"
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/cookiejar"
|
|
||||||
"net/url"
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
"path"
|
"path"
|
||||||
@@ -35,6 +32,15 @@ import (
|
|||||||
|
|
||||||
const readBuf = 32 * 1024
|
const readBuf = 32 * 1024
|
||||||
|
|
||||||
|
// minSplitSize is the smallest piece a file is split into: a segment shorter
|
||||||
|
// than this is not worth its own connection, so makeSegments stops dividing once
|
||||||
|
// the pieces would fall below it.
|
||||||
|
const minSplitSize = 20 << 20
|
||||||
|
|
||||||
|
// userAgent is sent on every request; there is no flag to change it. A site that
|
||||||
|
// needs a different one can be given a User-Agent header via --header.
|
||||||
|
const userAgent = "got/1.0"
|
||||||
|
|
||||||
// ErrOutputExists reports that the resolved output file already exists and
|
// ErrOutputExists reports that the resolved output file already exists and
|
||||||
// neither --allow-overwrite nor -c was given. main maps it to a distinct exit
|
// neither --allow-overwrite nor -c was given. main maps it to a distinct exit
|
||||||
// code via errors.Is, so it must stay on the error chain (CONTRACT).
|
// code via errors.Is, so it must stay on the error chain (CONTRACT).
|
||||||
@@ -45,60 +51,21 @@ var ErrOutputExists = errors.New("output file already exists")
|
|||||||
// (CONTRACT).
|
// (CONTRACT).
|
||||||
var ErrNotFound = errors.New("not found")
|
var ErrNotFound = errors.New("not found")
|
||||||
|
|
||||||
// ErrTooSlow reports a transfer aborted by --lowest-speed-limit. main maps it to
|
|
||||||
// exit code 5 via errors.Is, so it must stay on the error chain (CONTRACT).
|
|
||||||
var ErrTooSlow = errors.New("download speed too low")
|
|
||||||
|
|
||||||
// Config holds everything one HTTP download needs. It is assembled by main from
|
// Config holds everything one HTTP download needs. It is assembled by main from
|
||||||
// the resolved CLI options.
|
// the resolved CLI options.
|
||||||
type Config struct {
|
type Config struct {
|
||||||
Dir string
|
Dir string
|
||||||
Out string
|
Out string
|
||||||
Split int // --split: total connections across all mirrors
|
Split int // --split: total connections across all mirrors
|
||||||
MaxConnPerServer int // --max-connection-per-server: per-host connection cap
|
MinSplit int64 // smallest piece to split into; 0 = minSplitSize default (internal, no flag)
|
||||||
MinSplit int64
|
Tries int // --max-tries: 0 = unlimited
|
||||||
Tries int // 0 = unlimited
|
|
||||||
Timeout time.Duration
|
Timeout time.Duration
|
||||||
FileAlloc string // none | prealloc | trunc | falloc
|
FileAlloc string // none | prealloc | trunc | falloc
|
||||||
Continue bool
|
Continue bool
|
||||||
AllowOverwrite bool
|
AllowOverwrite bool
|
||||||
AutoRename bool
|
AutoRename bool
|
||||||
Headers []string
|
Headers []string // --header: extra request headers
|
||||||
UserAgent string
|
|
||||||
Referer string
|
|
||||||
Proxy string
|
|
||||||
CheckCert bool
|
|
||||||
Limit int64 // per-download bytes/s, 0 = unlimited
|
|
||||||
OverallLimiter *rate.Limiter // shared across all downloads, may be nil
|
OverallLimiter *rate.Limiter // shared across all downloads, may be nil
|
||||||
|
|
||||||
// RetryWait is the pause between retries of a transient failure
|
|
||||||
// (--retry-wait). 0 means retry immediately with no sleep.
|
|
||||||
RetryWait time.Duration
|
|
||||||
// AutoSaveInterval controls how often the .got control file is rewritten
|
|
||||||
// (--auto-save-interval). <=0 falls back to 60s.
|
|
||||||
AutoSaveInterval time.Duration
|
|
||||||
|
|
||||||
// ConnectTimeout is the time allowed to establish the TCP/TLS connection
|
|
||||||
// (--connect-timeout, default 60s), distinct from Timeout which
|
|
||||||
// covers stalls once data is flowing. <=0 falls back to Timeout.
|
|
||||||
ConnectTimeout time.Duration
|
|
||||||
// LoadCookies / SaveCookies are paths to a Netscape cookies.txt read into
|
|
||||||
// the client's jar before the download and written back on exit.
|
|
||||||
LoadCookies string
|
|
||||||
SaveCookies string
|
|
||||||
// ConditionalGet sends If-Modified-Since (from the existing local file's
|
|
||||||
// mtime) so a 304 reports success without re-downloading
|
|
||||||
// (--conditional-get).
|
|
||||||
ConditionalGet bool
|
|
||||||
// RemoteTime sets the finished file's mtime from the server's Last-Modified
|
|
||||||
// header (-R / --remote-time).
|
|
||||||
RemoteTime bool
|
|
||||||
// HTTPUser / HTTPPasswd enable HTTP Basic auth (--http-user/passwd).
|
|
||||||
HTTPUser string
|
|
||||||
HTTPPasswd string
|
|
||||||
// LowestSpeedLimit aborts a download whose speed stays at or below this many
|
|
||||||
// bytes/sec for a sustained window (--lowest-speed-limit). 0 = off.
|
|
||||||
LowestSpeedLimit int64
|
|
||||||
// Checksum is a "<type>=<digest>" spec (e.g. "sha-256=ab…"); when set,
|
// Checksum is a "<type>=<digest>" spec (e.g. "sha-256=ab…"); when set,
|
||||||
// the finished file is hashed and compared, failing with ErrChecksum on a
|
// the finished file is hashed and compared, failing with ErrChecksum on a
|
||||||
// mismatch (--checksum). Empty means no verification.
|
// mismatch (--checksum). Empty means no verification.
|
||||||
@@ -106,11 +73,6 @@ type Config struct {
|
|||||||
// DryRun probes the resource — proving it exists and its size — but downloads
|
// DryRun probes the resource — proving it exists and its size — but downloads
|
||||||
// nothing, then reports success (--dry-run).
|
// nothing, then reports success (--dry-run).
|
||||||
DryRun bool
|
DryRun bool
|
||||||
// CACert is a path to a PEM bundle of CA certificates used to verify HTTPS
|
|
||||||
// servers (--ca-certificate). Empty uses the system roots.
|
|
||||||
CACert string
|
|
||||||
// DisableIPv6 forces IPv4-only dialing (--disable-ipv6).
|
|
||||||
DisableIPv6 bool
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Download implements download.Download for one HTTP(S) file, fetched from one
|
// Download implements download.Download for one HTTP(S) file, fetched from one
|
||||||
@@ -119,26 +81,21 @@ type Config struct {
|
|||||||
// connections.
|
// connections.
|
||||||
type Download struct {
|
type Download struct {
|
||||||
uris []string // mirror list; uris[0] is the primary (naming + resume key)
|
uris []string // mirror list; uris[0] is the primary (naming + resume key)
|
||||||
maxConns int // segment workers = min(split, len(uris)*max-connection-per-server)
|
maxConns int // segment-worker cap = min(split, len(uris))
|
||||||
cfg Config
|
cfg Config
|
||||||
client *http.Client
|
client *http.Client
|
||||||
limit []*rate.Limiter
|
limit []*rate.Limiter
|
||||||
jar *cookiejar.Jar // non-nil when cookies are in use, for save-cookies
|
|
||||||
|
|
||||||
// name and out are resolved in Run after the probe response is known (the
|
// name and out are resolved in Run after the probe response is known (the
|
||||||
// final post-redirect URL and any Content-Disposition can change them), then
|
// final post-redirect URL and any Content-Disposition can change them). The
|
||||||
// never mutated again, so Stat can read name from another goroutine without a
|
// engine publishes a Download to the progress reporter before Run starts, so
|
||||||
// lock once Run has set it. They are seeded in New so a Stat before Run still
|
// Stat reads these from the reporter goroutine while resolveName writes them;
|
||||||
// has a sensible best-effort name.
|
// they are atomic.Pointer[string] (exactly like bt.Download.name) to keep that
|
||||||
name string
|
// read/write race-free. Seeded in New so a Stat before Run has a best-effort name.
|
||||||
out string
|
name atomic.Pointer[string]
|
||||||
|
out atomic.Pointer[string]
|
||||||
initErr error
|
initErr error
|
||||||
|
|
||||||
// mu guards seenURLs, which records every URL we sent a request to so
|
|
||||||
// SaveCookies can ask the jar for the cookies of each contacted host.
|
|
||||||
mu sync.Mutex
|
|
||||||
seenURLs []*url.URL
|
|
||||||
|
|
||||||
// live counters, read by Stat from any goroutine
|
// live counters, read by Stat from any goroutine
|
||||||
total int64 // -1 until known
|
total int64 // -1 until known
|
||||||
completed int64
|
completed int64
|
||||||
@@ -146,21 +103,6 @@ type Download struct {
|
|||||||
conns int32 // active connections
|
conns int32 // active connections
|
||||||
}
|
}
|
||||||
|
|
||||||
// loadCAPool reads a PEM bundle of CA certificates for verifying HTTPS servers
|
|
||||||
// (--ca-certificate). An unreadable file or one containing no
|
|
||||||
// certificates is an error rather than a silent fall-back to the system roots.
|
|
||||||
func loadCAPool(path string) (*x509.CertPool, error) {
|
|
||||||
pem, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
pool := x509.NewCertPool()
|
|
||||||
if !pool.AppendCertsFromPEM(pem) {
|
|
||||||
return nil, fmt.Errorf("ca-certificate %s: no certificates found", path)
|
|
||||||
}
|
|
||||||
return pool, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// primary is the canonical URL — used for naming and the resume sidecar so they
|
// primary is the canonical URL — used for naming and the resume sidecar so they
|
||||||
// stay stable regardless of which mirror served the bytes.
|
// stay stable regardless of which mirror served the bytes.
|
||||||
func (d *Download) primary() string { return d.uris[0] }
|
func (d *Download) primary() string { return d.uris[0] }
|
||||||
@@ -173,82 +115,27 @@ func (d *Download) mirror(n int) string { return d.uris[n%len(d.uris)] }
|
|||||||
// New builds an HTTP download for uris, which must all serve identical content
|
// New builds an HTTP download for uris, which must all serve identical content
|
||||||
// (mirrors). uris[0] is the primary.
|
// (mirrors). uris[0] is the primary.
|
||||||
func New(uris []string, cfg Config) *Download {
|
func New(uris []string, cfg Config) *Download {
|
||||||
// --connect-timeout bounds only connection establishment; once data flows
|
|
||||||
// --timeout (the idle guard) takes over. The dial and the TLS handshake each
|
|
||||||
// get the full connectTimeout (Go applies it per phase), so an HTTPS connect
|
|
||||||
// can take up to ~2x connectTimeout in the worst case — there is no single
|
|
||||||
// shared connect budget.
|
|
||||||
connectTimeout := cfg.ConnectTimeout
|
|
||||||
if connectTimeout <= 0 {
|
|
||||||
connectTimeout = cfg.Timeout
|
|
||||||
}
|
|
||||||
var initErr error
|
var initErr error
|
||||||
|
if cfg.MinSplit <= 0 {
|
||||||
// --ca-certificate: verify HTTPS servers against a custom CA bundle instead of
|
cfg.MinSplit = minSplitSize
|
||||||
// the system roots. A bad bundle becomes an initErr so Run fails loudly.
|
|
||||||
tlsCfg := &tls.Config{InsecureSkipVerify: !cfg.CheckCert}
|
|
||||||
if cfg.CACert != "" {
|
|
||||||
if pool, err := loadCAPool(cfg.CACert); err != nil {
|
|
||||||
initErr = err
|
|
||||||
} else {
|
|
||||||
tlsCfg.RootCAs = pool
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// --disable-ipv6: force IPv4 by dialing "tcp4"; a plain "tcp" would let the
|
// One connect budget: --timeout bounds both establishing the connection and
|
||||||
// resolver hand back a v6 address.
|
// idle stalls once data flows. Proxies still come from the standard
|
||||||
dialer := &net.Dialer{Timeout: connectTimeout}
|
// HTTP_PROXY/HTTPS_PROXY environment, and TLS uses the system roots.
|
||||||
dialContext := dialer.DialContext
|
|
||||||
if cfg.DisableIPv6 {
|
|
||||||
dialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
|
|
||||||
if network == "tcp" {
|
|
||||||
network = "tcp4"
|
|
||||||
}
|
|
||||||
return dialer.DialContext(ctx, network, addr)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// MaxConnsPerHost is the per-host cap (--max-connection-per-server): with
|
|
||||||
// mirrors the total worker count can exceed it, but no single host does.
|
|
||||||
perHost := cfg.MaxConnPerServer
|
|
||||||
if perHost < 1 {
|
|
||||||
perHost = 1
|
|
||||||
}
|
|
||||||
tr := &http.Transport{
|
tr := &http.Transport{
|
||||||
Proxy: http.ProxyFromEnvironment,
|
Proxy: http.ProxyFromEnvironment,
|
||||||
MaxConnsPerHost: perHost,
|
MaxConnsPerHost: cfg.Split,
|
||||||
MaxIdleConnsPerHost: perHost,
|
MaxIdleConnsPerHost: cfg.Split,
|
||||||
ResponseHeaderTimeout: cfg.Timeout,
|
ResponseHeaderTimeout: cfg.Timeout,
|
||||||
TLSHandshakeTimeout: connectTimeout,
|
TLSHandshakeTimeout: cfg.Timeout,
|
||||||
DialContext: dialContext,
|
DialContext: (&net.Dialer{Timeout: cfg.Timeout}).DialContext,
|
||||||
TLSClientConfig: tlsCfg,
|
|
||||||
}
|
|
||||||
if cfg.Proxy != "" {
|
|
||||||
if pu, err := url.Parse(cfg.Proxy); err == nil {
|
|
||||||
tr.Proxy = http.ProxyURL(pu)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Seed the cookie jar from --load-cookies; a load error becomes an initErr so
|
|
||||||
// Run surfaces it rather than silently downloading uncredentialed.
|
|
||||||
var jar *cookiejar.Jar
|
|
||||||
if cfg.LoadCookies != "" || cfg.SaveCookies != "" {
|
|
||||||
if j, err := loadCookieJar(cfg.LoadCookies); err != nil {
|
|
||||||
if initErr == nil {
|
|
||||||
initErr = err
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
jar = j
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
var limit []*rate.Limiter
|
var limit []*rate.Limiter
|
||||||
if cfg.OverallLimiter != nil {
|
if cfg.OverallLimiter != nil {
|
||||||
limit = append(limit, cfg.OverallLimiter)
|
limit = append(limit, cfg.OverallLimiter)
|
||||||
}
|
}
|
||||||
if cfg.Limit > 0 {
|
|
||||||
limit = append(limit, rate.NewLimiter(rate.Limit(cfg.Limit), download.LimiterBurst(cfg.Limit)))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Seed a best-effort name from -o or the primary URL. The real name is
|
// Seed a best-effort name from -o or the primary URL. The real name is
|
||||||
// finalised in Run after probe(), where Content-Disposition and the
|
// finalised in Run after probe(), where Content-Disposition and the
|
||||||
@@ -268,47 +155,54 @@ func New(uris []string, cfg Config) *Download {
|
|||||||
if len(uris) == 0 && initErr == nil {
|
if len(uris) == 0 && initErr == nil {
|
||||||
initErr = errors.New("no URL to download")
|
initErr = errors.New("no URL to download")
|
||||||
}
|
}
|
||||||
// Worker count: --split connections are spread across the mirrors, capped
|
// Worker count is --split: the file is cut into that many byte-range segments,
|
||||||
// at --max-connection-per-server per host, so the ceiling is
|
// each its own connection, whether served by one host or spread across mirrors.
|
||||||
// min(split, M*max-conn-per-server). One mirror reduces to min(split, x).
|
|
||||||
maxConns := cfg.Split
|
maxConns := cfg.Split
|
||||||
if m := max(1, len(uris)) * perHost; maxConns > m {
|
|
||||||
maxConns = m
|
|
||||||
}
|
|
||||||
if maxConns < 1 {
|
if maxConns < 1 {
|
||||||
maxConns = 1
|
maxConns = 1
|
||||||
}
|
}
|
||||||
|
|
||||||
// Only attach the jar when one was built: a typed-nil *cookiejar.Jar stored
|
d := &Download{
|
||||||
// in the http.Client.Jar interface is non-nil and would panic on use.
|
|
||||||
client := &http.Client{Transport: tr}
|
|
||||||
if jar != nil {
|
|
||||||
client.Jar = jar
|
|
||||||
}
|
|
||||||
|
|
||||||
return &Download{
|
|
||||||
uris: uris,
|
uris: uris,
|
||||||
maxConns: maxConns,
|
maxConns: maxConns,
|
||||||
cfg: cfg,
|
cfg: cfg,
|
||||||
client: client,
|
client: &http.Client{Transport: tr},
|
||||||
limit: limit,
|
limit: limit,
|
||||||
jar: jar,
|
|
||||||
name: name,
|
|
||||||
out: out,
|
|
||||||
total: -1,
|
total: -1,
|
||||||
initErr: initErr,
|
initErr: initErr,
|
||||||
}
|
}
|
||||||
|
d.setName(name)
|
||||||
|
d.setOut(out)
|
||||||
|
return d
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d *Download) Name() string { return d.name }
|
func (d *Download) Name() string { return d.loadName() }
|
||||||
|
|
||||||
// Path returns the resolved output file path (used by --follow-torrent).
|
// Path returns the resolved output file path, so a fetched .torrent can be
|
||||||
func (d *Download) Path() string { return d.out }
|
// followed into a BitTorrent download of its content.
|
||||||
|
func (d *Download) Path() string { return d.loadOut() }
|
||||||
|
|
||||||
|
// name and out are read by Stat from the reporter goroutine while Run resolves
|
||||||
|
// them, so they are stored atomically (mirroring bt.Download.name).
|
||||||
|
func (d *Download) setName(s string) { d.name.Store(&s) }
|
||||||
|
func (d *Download) setOut(s string) { d.out.Store(&s) }
|
||||||
|
func (d *Download) loadName() string {
|
||||||
|
if p := d.name.Load(); p != nil {
|
||||||
|
return *p
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
func (d *Download) loadOut() string {
|
||||||
|
if p := d.out.Load(); p != nil {
|
||||||
|
return *p
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
func (d *Download) Stat() download.Stat {
|
func (d *Download) Stat() download.Stat {
|
||||||
return download.Stat{
|
return download.Stat{
|
||||||
Name: d.name,
|
Name: d.loadName(),
|
||||||
ID: d.out, // resolved output path; stable per process (CONTRACT)
|
ID: d.loadOut(), // resolved output path; stable per process (CONTRACT)
|
||||||
IsBT: false,
|
IsBT: false,
|
||||||
Status: download.Status(atomic.LoadInt32(&d.status)),
|
Status: download.Status(atomic.LoadInt32(&d.status)),
|
||||||
Total: atomic.LoadInt64(&d.total),
|
Total: atomic.LoadInt64(&d.total),
|
||||||
@@ -332,7 +226,6 @@ type probeResult struct {
|
|||||||
lastmod string
|
lastmod string
|
||||||
cdisp string // raw Content-Disposition header
|
cdisp string // raw Content-Disposition header
|
||||||
finalURL *url.URL
|
finalURL *url.URL
|
||||||
notModified bool // server answered 304 to a conditional GET
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Run probes the resource, finalises the output name, decides between a single
|
// Run probes the resource, finalises the output name, decides between a single
|
||||||
@@ -355,12 +248,6 @@ func (d *Download) Run(ctx context.Context) (err error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
// --conditional-get: the server said our local copy is current, so there is
|
|
||||||
// nothing to download. Report success without touching the file.
|
|
||||||
if pr.notModified {
|
|
||||||
d.setStatus(download.Complete)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
atomic.StoreInt64(&d.total, pr.total)
|
atomic.StoreInt64(&d.total, pr.total)
|
||||||
|
|
||||||
// Finalise name/out now that the post-redirect URL and Content-Disposition
|
// Finalise name/out now that the post-redirect URL and Content-Disposition
|
||||||
@@ -388,36 +275,22 @@ func (d *Download) Run(ctx context.Context) (err error) {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// --lowest-speed-limit: cancel the transfer if its speed stays at or below
|
out := d.loadOut()
|
||||||
// the limit past a startup grace. The watcher cancels xferCtx and sets a
|
|
||||||
// flag so we can return the distinct "too slow" error rather than a bare
|
|
||||||
// cancellation.
|
|
||||||
xferCtx, stopGuard, tooSlow := d.speedGuard(ctx)
|
|
||||||
if !pr.ranged || pr.total <= 0 {
|
if !pr.ranged || pr.total <= 0 {
|
||||||
err = d.single(xferCtx, d.out)
|
err = d.single(ctx, out)
|
||||||
} else {
|
} else {
|
||||||
err = d.segmented(xferCtx, d.out, pr.total, pr.etag, pr.lastmod)
|
err = d.segmented(ctx, out, pr.total, pr.etag, pr.lastmod)
|
||||||
}
|
}
|
||||||
stopGuard()
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if tooSlow() {
|
|
||||||
return fmt.Errorf("%s: %w (<= %d bytes/sec)", d.primary(), ErrTooSlow, d.cfg.LowestSpeedLimit)
|
|
||||||
}
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
// --checksum: verify the finished file before declaring success, so a
|
// --checksum: verify the finished file before declaring success, so a
|
||||||
// corrupted or tampered download fails (--checksum, exit code 32).
|
// corrupted or tampered download fails (--checksum, exit code 32).
|
||||||
if newSum != nil {
|
if newSum != nil {
|
||||||
if err = verifyFile(d.out, newSum, sumWant); err != nil {
|
if err = verifyFile(out, newSum, sumWant); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// -R / --remote-time: stamp the finished file with the server's mtime.
|
|
||||||
if d.cfg.RemoteTime && pr.lastmod != "" {
|
|
||||||
if t, err := http.ParseTime(pr.lastmod); err == nil {
|
|
||||||
os.Chtimes(d.out, t, t)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
d.setStatus(download.Complete)
|
d.setStatus(download.Complete)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -427,6 +300,7 @@ func (d *Download) Run(ctx context.Context) (err error) {
|
|||||||
// the original URL. Overwrite / auto-rename is then resolved against the final
|
// the original URL. Overwrite / auto-rename is then resolved against the final
|
||||||
// path, honouring -c so a resumable file is never renamed (items [2],[8],[13]).
|
// path, honouring -c so a resumable file is never renamed (items [2],[8],[13]).
|
||||||
func (d *Download) resolveName(pr probeResult) error {
|
func (d *Download) resolveName(pr probeResult) error {
|
||||||
|
out := d.loadOut()
|
||||||
if d.cfg.Out == "" {
|
if d.cfg.Out == "" {
|
||||||
name := ""
|
name := ""
|
||||||
if cd := pr.cdisp; cd != "" {
|
if cd := pr.cdisp; cd != "" {
|
||||||
@@ -438,8 +312,9 @@ func (d *Download) resolveName(pr probeResult) error {
|
|||||||
if name == "" {
|
if name == "" {
|
||||||
name = nameFromURL(d.primary())
|
name = nameFromURL(d.primary())
|
||||||
}
|
}
|
||||||
d.name = name
|
out = filepath.Join(d.cfg.Dir, name)
|
||||||
d.out = filepath.Join(d.cfg.Dir, name)
|
d.setName(name)
|
||||||
|
d.setOut(out)
|
||||||
}
|
}
|
||||||
|
|
||||||
// --dry-run writes nothing, so skip overwrite/auto-rename resolution; the name
|
// --dry-run writes nothing, so skip overwrite/auto-rename resolution; the name
|
||||||
@@ -450,38 +325,24 @@ func (d *Download) resolveName(pr probeResult) error {
|
|||||||
|
|
||||||
// -c resumes either our own .got sidecar or a foreign/browser partial file,
|
// -c resumes either our own .got sidecar or a foreign/browser partial file,
|
||||||
// so an existing file is never auto-renamed under -c (item [2]).
|
// so an existing file is never auto-renamed under -c (item [2]).
|
||||||
if d.cfg.Continue && fileExists(d.out) {
|
if d.cfg.Continue && fileExists(out) {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
if fileExists(d.out) && !d.cfg.AllowOverwrite {
|
if fileExists(out) && !d.cfg.AllowOverwrite {
|
||||||
if d.cfg.AutoRename {
|
if d.cfg.AutoRename {
|
||||||
u, err := uniqueName(d.out)
|
u, err := uniqueName(out)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
d.out = u
|
d.setOut(u)
|
||||||
d.name = filepath.Base(u)
|
d.setName(filepath.Base(u))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
return fmt.Errorf("%s: %w (use --allow-overwrite or -c)", d.out, ErrOutputExists)
|
return fmt.Errorf("%s: %w (use --allow-overwrite or -c)", out, ErrOutputExists)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// backoff waits RetryWait between attempts, but never after the final attempt,
|
|
||||||
// and returns ctx.Err() if cancelled while waiting (item [20]).
|
|
||||||
func backoff(ctx context.Context, wait time.Duration, tries, attempt int) error {
|
|
||||||
if wait <= 0 || (tries != 0 && attempt >= tries-1) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return ctx.Err()
|
|
||||||
case <-time.After(wait):
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// retriesExhausted wraps the final cause once the retry budget is spent, so the
|
// retriesExhausted wraps the final cause once the retry budget is spent, so the
|
||||||
// caller — and the exit-code mapping in main — can still see whether it was DNS,
|
// caller — and the exit-code mapping in main — can still see whether it was DNS,
|
||||||
// a refused connection, or a timeout, rather than an opaque "too many retries".
|
// a refused connection, or a timeout, rather than an opaque "too many retries".
|
||||||
@@ -499,12 +360,11 @@ func retriesExhausted(what string, tries int, cause error) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// withRetries runs attempt up to d.cfg.Tries times (0 = unlimited), the shared
|
// withRetries runs attempt up to d.cfg.Tries times (0 = unlimited), the shared
|
||||||
// tries/backoff/fatal policy that probe, the single stream, and each segment all
|
// tries/fatal policy that probe, the single stream, and each segment all use so
|
||||||
// use so one transient DNS/connect/5xx does not fail the whole download (item
|
// one transient DNS/connect/5xx does not fail the whole download (item [18]). It
|
||||||
// [18]). It returns immediately on success (nil), on context cancellation, or on
|
// returns immediately on success (nil), on context cancellation, or on a fatal{}
|
||||||
// a fatal{} error (retrying cannot help), sleeps RetryWait between attempts, and
|
// error (retrying cannot help), and wraps the final cause with retriesExhausted
|
||||||
// wraps the final cause with retriesExhausted(what, ...) once the budget is
|
// once the budget is spent so the underlying cause stays inspectable.
|
||||||
// spent so the underlying cause stays inspectable.
|
|
||||||
func (d *Download) withRetries(ctx context.Context, what string, attempt func() error) error {
|
func (d *Download) withRetries(ctx context.Context, what string, attempt func() error) error {
|
||||||
tries := d.cfg.Tries
|
tries := d.cfg.Tries
|
||||||
var lastErr error
|
var lastErr error
|
||||||
@@ -521,37 +381,48 @@ func (d *Download) withRetries(ctx context.Context, what string, attempt func()
|
|||||||
if errors.As(err, &fe) {
|
if errors.As(err, &fe) {
|
||||||
return err // permanent: retrying cannot help
|
return err // permanent: retrying cannot help
|
||||||
}
|
}
|
||||||
if err := backoff(ctx, d.cfg.RetryWait, tries, n); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return retriesExhausted(what, tries, lastErr)
|
return retriesExhausted(what, tries, lastErr)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// overMirrors calls fn against each mirror URL in turn, starting at offset start
|
||||||
|
// so callers can spread work across mirrors (segment index) and rotate on retry.
|
||||||
|
// It returns nil on the first success, ctx.Err() if cancelled between mirrors, or
|
||||||
|
// the last error once every mirror is spent.
|
||||||
|
func (d *Download) overMirrors(ctx context.Context, start int, fn func(uri string) error) error {
|
||||||
|
var lastErr error
|
||||||
|
for i := range d.uris {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
if err := fn(d.mirror(start + i)); err != nil {
|
||||||
|
lastErr = err
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return lastErr
|
||||||
|
}
|
||||||
|
|
||||||
// probeRetry runs probe within the shared retry policy (item [18]). The probe
|
// probeRetry runs probe within the shared retry policy (item [18]). The probe
|
||||||
// result is captured through pr because withRetries only carries an error.
|
// result is captured through pr because withRetries only carries an error.
|
||||||
func (d *Download) probeRetry(ctx context.Context) (probeResult, error) {
|
func (d *Download) probeRetry(ctx context.Context) (probeResult, error) {
|
||||||
// Probe mirrors in order (primary first); the first that answers anchors the
|
// Probe mirrors in order (primary first); the first that answers anchors the
|
||||||
// size and validators for the whole download. A dead/404 primary falls over
|
// size and validators for the whole download. A dead/404 primary falls over
|
||||||
// to the next mirror.
|
// to the next mirror.
|
||||||
var lastErr error
|
|
||||||
for _, u := range d.uris {
|
|
||||||
if ctx.Err() != nil {
|
|
||||||
return probeResult{}, ctx.Err()
|
|
||||||
}
|
|
||||||
var pr probeResult
|
var pr probeResult
|
||||||
err := d.withRetries(ctx, u, func() error {
|
err := d.overMirrors(ctx, 0, func(uri string) error {
|
||||||
|
return d.withRetries(ctx, uri, func() error {
|
||||||
var perr error
|
var perr error
|
||||||
pr, perr = d.probe(ctx, u)
|
pr, perr = d.probe(ctx, uri)
|
||||||
return perr
|
return perr
|
||||||
})
|
})
|
||||||
if err == nil {
|
})
|
||||||
|
if err != nil {
|
||||||
|
return probeResult{}, err
|
||||||
|
}
|
||||||
return pr, nil
|
return pr, nil
|
||||||
}
|
}
|
||||||
lastErr = err
|
|
||||||
}
|
|
||||||
return probeResult{}, lastErr
|
|
||||||
}
|
|
||||||
|
|
||||||
// probe issues a one-byte ranged GET to learn the total size, whether the
|
// probe issues a one-byte ranged GET to learn the total size, whether the
|
||||||
// server honours ranges, and the validator headers used for safe resume. It
|
// server honours ranges, and the validator headers used for safe resume. It
|
||||||
@@ -561,15 +432,6 @@ func (d *Download) probe(ctx context.Context, uri string) (probeResult, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return probeResult{}, err
|
return probeResult{}, err
|
||||||
}
|
}
|
||||||
// --conditional-get: ask the server to skip the transfer when our local file
|
|
||||||
// is already current. Only done when there is no .got control file (an
|
|
||||||
// in-progress resume must not be short-circuited), using the local file's
|
|
||||||
// mtime for If-Modified-Since.
|
|
||||||
if d.cfg.ConditionalGet && !fileExists(controlPath(d.out)) {
|
|
||||||
if fi, err := os.Stat(d.out); err == nil {
|
|
||||||
req.Header.Set("If-Modified-Since", fi.ModTime().UTC().Format(http.TimeFormat))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
resp, err := d.client.Do(req)
|
resp, err := d.client.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return probeResult{}, err
|
return probeResult{}, err
|
||||||
@@ -586,11 +448,6 @@ func (d *Download) probe(ctx context.Context, uri string) (probeResult, error) {
|
|||||||
pr.finalURL = resp.Request.URL
|
pr.finalURL = resp.Request.URL
|
||||||
}
|
}
|
||||||
|
|
||||||
if resp.StatusCode == http.StatusNotModified {
|
|
||||||
pr.notModified = true
|
|
||||||
return pr, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
switch resp.StatusCode {
|
switch resp.StatusCode {
|
||||||
case http.StatusPartialContent:
|
case http.StatusPartialContent:
|
||||||
// Content-Range: bytes 0-0/12345
|
// Content-Range: bytes 0-0/12345
|
||||||
@@ -652,37 +509,23 @@ func (d *Download) segmented(ctx context.Context, out string, total int64, etag,
|
|||||||
saveDone := make(chan struct{})
|
saveDone := make(chan struct{})
|
||||||
go d.saveLoop(ctx, out, total, etag, lastmod, segs, saveDone)
|
go d.saveLoop(ctx, out, total, etag, lastmod, segs, saveDone)
|
||||||
|
|
||||||
jobs := make(chan *seg)
|
// One worker per segment. Segments are non-overlapping byte ranges written
|
||||||
go func() {
|
// with WriteAt, so the workers share no cursor and need no coordination — a
|
||||||
defer close(jobs)
|
// finished worker simply exits. The division (makeSegments / a resumed
|
||||||
for i := range segs {
|
// control) fixes the parallelism up front; there is no rebalancing.
|
||||||
if segs[i].done() {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case jobs <- &segs[i]:
|
|
||||||
case <-ctx.Done(): // stop feeding once we're tearing down
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
var (
|
var (
|
||||||
wg sync.WaitGroup
|
wg sync.WaitGroup
|
||||||
errOnce sync.Once
|
errOnce sync.Once
|
||||||
runErr error
|
runErr error
|
||||||
)
|
)
|
||||||
for w := 0; w < d.maxConns; w++ {
|
for i := range segs {
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
go func() {
|
go func(s *seg) {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
for s := range jobs {
|
|
||||||
if err := d.fetchSeg(ctx, f, s, total); err != nil {
|
if err := d.fetchSeg(ctx, f, s, total); err != nil {
|
||||||
errOnce.Do(func() { runErr = err; cancel() })
|
errOnce.Do(func() { runErr = err; cancel() })
|
||||||
return
|
|
||||||
}
|
}
|
||||||
}
|
}(&segs[i])
|
||||||
}()
|
|
||||||
}
|
}
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
cancel()
|
cancel()
|
||||||
@@ -706,24 +549,14 @@ func (d *Download) fetchSeg(ctx context.Context, f *os.File, s *seg, total int64
|
|||||||
// The segment fails only once every mirror is spent. Starting at s.index
|
// The segment fails only once every mirror is spent. Starting at s.index
|
||||||
// spreads the segments across mirrors round-robin, and each attempt resumes
|
// spreads the segments across mirrors round-robin, and each attempt resumes
|
||||||
// from s.offset().
|
// from s.offset().
|
||||||
var lastErr error
|
return d.overMirrors(ctx, s.index, func(uri string) error {
|
||||||
for i := range d.uris {
|
return d.withRetries(ctx, fmt.Sprintf("segment %d", s.index), func() error {
|
||||||
if ctx.Err() != nil {
|
|
||||||
return ctx.Err()
|
|
||||||
}
|
|
||||||
uri := d.mirror(s.index + i)
|
|
||||||
err := d.withRetries(ctx, fmt.Sprintf("segment %d", s.index), func() error {
|
|
||||||
if s.done() {
|
if s.done() {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
return d.fetchOnce(ctx, f, s, uri, total)
|
return d.fetchOnce(ctx, f, s, uri, total)
|
||||||
})
|
})
|
||||||
if err == nil {
|
})
|
||||||
return nil
|
|
||||||
}
|
|
||||||
lastErr = err
|
|
||||||
}
|
|
||||||
return lastErr
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d *Download) fetchOnce(ctx context.Context, f *os.File, s *seg, uri string, total int64) error {
|
func (d *Download) fetchOnce(ctx context.Context, f *os.File, s *seg, uri string, total int64) error {
|
||||||
@@ -744,129 +577,95 @@ func (d *Download) fetchOnce(ctx context.Context, f *os.File, s *seg, uri string
|
|||||||
atomic.AddInt32(&d.conns, -1)
|
atomic.AddInt32(&d.conns, -1)
|
||||||
}()
|
}()
|
||||||
if resp.StatusCode != http.StatusPartialContent {
|
if resp.StatusCode != http.StatusPartialContent {
|
||||||
return statusError(fmt.Sprintf("segment %d: expected 206, got %s", s.index, resp.Status), resp.StatusCode)
|
err := statusError(fmt.Sprintf("segment %d: expected 206, got %s", s.index, resp.Status), resp.StatusCode)
|
||||||
|
// A 200 means this mirror ignores Range entirely; retrying it only burns the
|
||||||
|
// budget, so make it fatal and let fetchSeg fail over to the next mirror.
|
||||||
|
if resp.StatusCode == http.StatusOK {
|
||||||
|
err = fatal{err}
|
||||||
|
}
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
// A mirror must serve the same file as the primary, or its bytes written at
|
// A mirror must serve the same file as the primary, or its bytes written at
|
||||||
// this offset would corrupt the output. Reject a 206 whose total length
|
// this offset would corrupt the output. Require a Content-Range that confirms
|
||||||
// disagrees with the probe (validate the total length); fatal so withRetries
|
// both the offset we asked for and the total length the probe saw; a missing,
|
||||||
// stops and fetchSeg falls over to the next mirror instead of writing it.
|
// unparseable, or divergent range is fatal so withRetries stops and fetchSeg
|
||||||
if _, _, t, ok := parseContentRange(resp.Header.Get("Content-Range")); ok && t != total {
|
// falls over to the next mirror instead of writing unverified bytes.
|
||||||
return fatal{fmt.Errorf("segment %d: %s reports length %d, want %d", s.index, uri, t, total)}
|
if start, _, t, ok := parseContentRange(resp.Header.Get("Content-Range")); !ok || t != total || start != s.offset() {
|
||||||
|
return fatal{fmt.Errorf("segment %d: %s returned an unverifiable range %q (want start %d, length %d)",
|
||||||
|
s.index, uri, resp.Header.Get("Content-Range"), s.offset(), total)}
|
||||||
}
|
}
|
||||||
body, stop := d.idleGuard(resp.Body, cancel)
|
body, stop := d.idleGuard(resp.Body, cancel)
|
||||||
defer stop()
|
defer stop()
|
||||||
return d.pump(ctx, f, s, body)
|
return d.pump(ctx, f, s, body)
|
||||||
}
|
}
|
||||||
|
|
||||||
// errIdleTimeout marks a read that stalled past the idle window, so callers can
|
// ErrTimeout marks a transfer that stalled past the idle window (--timeout): a
|
||||||
// report a distinct "timed out" instead of the bare "context canceled" that the
|
// distinct "timed out" instead of the bare "context canceled" the cancelled
|
||||||
// cancelled request would otherwise surface (item [39]).
|
// request would otherwise surface (item [39]). main maps it to exit code 2 via
|
||||||
var errIdleTimeout = errors.New("idle timeout: no data received")
|
// errors.Is, so it must stay on the error chain (CONTRACT).
|
||||||
|
var ErrTimeout = errors.New("idle timeout: no data received")
|
||||||
|
|
||||||
// idleReader resets a watchdog timer on every read; if a read stalls longer
|
// idleReader records the time of the last byte received; a background watcher
|
||||||
// than the idle window the timer fires and cancels the request, so the blocked
|
// (idleGuard) cancels the request once a full idle window passes with no
|
||||||
// Read returns an error instead of hanging forever on a wedged connection.
|
// progress. Tracking progress by timestamp — rather than arming a per-read timer
|
||||||
|
// — means a slow-but-advancing transfer is never aborted, and there is no
|
||||||
|
// Reset/Stop race that could poison a connection that was still delivering data.
|
||||||
type idleReader struct {
|
type idleReader struct {
|
||||||
r io.Reader
|
r io.Reader
|
||||||
timer *time.Timer
|
|
||||||
idle time.Duration
|
idle time.Duration
|
||||||
fired atomic.Bool // set when the watchdog cancelled the request
|
last atomic.Int64 // UnixNano of the last byte received
|
||||||
|
timedOut atomic.Bool // set when the watcher cancelled the request
|
||||||
}
|
}
|
||||||
|
|
||||||
func (ir *idleReader) Read(p []byte) (int, error) {
|
func (ir *idleReader) Read(p []byte) (int, error) {
|
||||||
ir.timer.Reset(ir.idle)
|
|
||||||
n, err := ir.r.Read(p)
|
n, err := ir.r.Read(p)
|
||||||
ir.timer.Stop() // only the Read itself counts as idle, not write/throttle gaps
|
if n > 0 {
|
||||||
if err != nil && ir.fired.Load() {
|
ir.last.Store(time.Now().UnixNano())
|
||||||
// The cancellation came from our watchdog, not the caller's ctx, so
|
}
|
||||||
|
if err != nil && ir.timedOut.Load() {
|
||||||
|
// The cancellation came from our watcher, not the caller's ctx, so
|
||||||
// translate the read error into the distinct idle-timeout sentinel.
|
// translate the read error into the distinct idle-timeout sentinel.
|
||||||
return n, fmt.Errorf("%w (after %s)", errIdleTimeout, ir.idle)
|
return n, fmt.Errorf("%w (after %s)", ErrTimeout, ir.idle)
|
||||||
}
|
}
|
||||||
return n, err
|
return n, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// idleGuard wraps body with an idle timeout of d.cfg.Timeout, cancelling the
|
// watch cancels the request once idle passes with no byte received. It polls a
|
||||||
// request if no data arrives in that window. It returns the reader to use and a
|
// few times per window so a stall is caught within ~idle of the last byte,
|
||||||
// stop func to call when the transfer is done.
|
// without a shared timer that Read would have to Reset/Stop.
|
||||||
|
func (ir *idleReader) watch(stop <-chan struct{}, cancel context.CancelFunc) {
|
||||||
|
tick := ir.idle / 4
|
||||||
|
if tick <= 0 {
|
||||||
|
tick = ir.idle
|
||||||
|
}
|
||||||
|
t := time.NewTicker(tick)
|
||||||
|
defer t.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-stop:
|
||||||
|
return
|
||||||
|
case now := <-t.C:
|
||||||
|
if now.UnixNano()-ir.last.Load() >= int64(ir.idle) {
|
||||||
|
ir.timedOut.Store(true)
|
||||||
|
cancel()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// idleGuard wraps body with an idle timeout of d.cfg.Timeout: a watcher goroutine
|
||||||
|
// cancels the request when no byte arrives for that long. It returns the reader to
|
||||||
|
// use and a stop func to call (exactly once) when the transfer is done.
|
||||||
func (d *Download) idleGuard(body io.Reader, cancel context.CancelFunc) (io.Reader, func()) {
|
func (d *Download) idleGuard(body io.Reader, cancel context.CancelFunc) (io.Reader, func()) {
|
||||||
if d.cfg.Timeout <= 0 {
|
if d.cfg.Timeout <= 0 {
|
||||||
return body, func() {}
|
return body, func() {}
|
||||||
}
|
}
|
||||||
ir := &idleReader{r: body, idle: d.cfg.Timeout}
|
ir := &idleReader{r: body, idle: d.cfg.Timeout}
|
||||||
ir.timer = time.AfterFunc(d.cfg.Timeout, func() {
|
ir.last.Store(time.Now().UnixNano())
|
||||||
ir.fired.Store(true)
|
stop := make(chan struct{})
|
||||||
cancel()
|
go ir.watch(stop, cancel)
|
||||||
})
|
return ir, func() { close(stop) }
|
||||||
return ir, func() { ir.timer.Stop() }
|
|
||||||
}
|
|
||||||
|
|
||||||
// speedStartupGrace is how long a download may stay slow before the
|
|
||||||
// lowest-speed-limit watchdog starts checking it: a startup idle time of
|
|
||||||
// 10 seconds.
|
|
||||||
const speedStartupGrace = 10 * time.Second
|
|
||||||
|
|
||||||
// belowSpeed reports whether the bytes transferred between two cumulative
|
|
||||||
// d.completed samples (prev -> now over one window) put the download at or
|
|
||||||
// below limit. A negative delta is not a slow second: d.completed is not
|
|
||||||
// strictly monotonic — singleOnce rewrites it with an absolute StoreInt64 and a
|
|
||||||
// resumed Range answered by a 200 resets the offset to 0, so the counter can
|
|
||||||
// jump backward on a retry. Treating that backward jump as "below limit" would
|
|
||||||
// false-abort a healthy download, so a reset (delta < 0) reports false; the
|
|
||||||
// caller has already advanced its baseline, so the next full window measures the
|
|
||||||
// real speed. Only a genuine non-negative delta at or under the limit aborts.
|
|
||||||
func belowSpeed(prev, now, limit int64) bool {
|
|
||||||
delta := now - prev
|
|
||||||
if delta < 0 {
|
|
||||||
return false // counter reset, not a slow window
|
|
||||||
}
|
|
||||||
return delta <= limit
|
|
||||||
}
|
|
||||||
|
|
||||||
// speedGuard implements --lowest-speed-limit. It returns a child context that
|
|
||||||
// the watcher cancels (setting a flag) once the measured download speed has
|
|
||||||
// stayed at or below the limit past the startup grace, plus a stop func and a
|
|
||||||
// predicate reporting whether the watcher fired. When the limit is 0 it is a
|
|
||||||
// no-op passthrough.
|
|
||||||
//
|
|
||||||
// Speed is sampled from d.completed over a one-second window: a single slow
|
|
||||||
// second after the grace aborts the download as too slow.
|
|
||||||
func (d *Download) speedGuard(ctx context.Context) (context.Context, func(), func() bool) {
|
|
||||||
if d.cfg.LowestSpeedLimit <= 0 {
|
|
||||||
return ctx, func() {}, func() bool { return false }
|
|
||||||
}
|
|
||||||
child, cancel := context.WithCancel(ctx)
|
|
||||||
var fired atomic.Bool
|
|
||||||
done := make(chan struct{})
|
|
||||||
go func() {
|
|
||||||
defer close(done)
|
|
||||||
start := time.Now()
|
|
||||||
ticker := time.NewTicker(time.Second)
|
|
||||||
defer ticker.Stop()
|
|
||||||
last := atomic.LoadInt64(&d.completed)
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-child.Done():
|
|
||||||
return
|
|
||||||
case <-ticker.C:
|
|
||||||
now := atomic.LoadInt64(&d.completed)
|
|
||||||
prev := last
|
|
||||||
last = now
|
|
||||||
if time.Since(start) < speedStartupGrace {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if belowSpeed(prev, now, d.cfg.LowestSpeedLimit) {
|
|
||||||
fired.Store(true)
|
|
||||||
cancel()
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
stop := func() {
|
|
||||||
cancel()
|
|
||||||
<-done
|
|
||||||
}
|
|
||||||
return child, stop, func() bool { return fired.Load() }
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// fatal marks an error that retrying cannot fix. Unwrap exposes the wrapped
|
// fatal marks an error that retrying cannot fix. Unwrap exposes the wrapped
|
||||||
@@ -901,7 +700,9 @@ func statusError(ctxMsg string, code int) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// pump copies the response body into the file at the segment's running offset,
|
// pump copies the response body into the file at the segment's running offset,
|
||||||
// stopping at the segment end and respecting rate limits.
|
// stopping at the segment end and respecting rate limits. Only this worker writes
|
||||||
|
// s.written, so the advance is a plain atomic add (Stat and the snapshot read it
|
||||||
|
// atomically); the loop ends when the range is filled.
|
||||||
func (d *Download) pump(ctx context.Context, f *os.File, s *seg, body io.Reader) error {
|
func (d *Download) pump(ctx context.Context, f *os.File, s *seg, body io.Reader) error {
|
||||||
buf := make([]byte, readBuf)
|
buf := make([]byte, readBuf)
|
||||||
for s.remaining() > 0 {
|
for s.remaining() > 0 {
|
||||||
@@ -914,7 +715,7 @@ func (d *Download) pump(ctx context.Context, f *os.File, s *seg, body io.Reader)
|
|||||||
if _, werr := f.WriteAt(buf[:rd], s.offset()); werr != nil {
|
if _, werr := f.WriteAt(buf[:rd], s.offset()); werr != nil {
|
||||||
return werr
|
return werr
|
||||||
}
|
}
|
||||||
s.advance(int64(rd))
|
s.addWritten(int64(rd))
|
||||||
atomic.AddInt64(&d.completed, int64(rd))
|
atomic.AddInt64(&d.completed, int64(rd))
|
||||||
d.throttle(ctx, rd)
|
d.throttle(ctx, rd)
|
||||||
}
|
}
|
||||||
@@ -953,23 +754,13 @@ func (d *Download) single(ctx context.Context, out string) error {
|
|||||||
first := true
|
first := true
|
||||||
// One connection at a time, but try each mirror: a mirror that fails its
|
// One connection at a time, but try each mirror: a mirror that fails its
|
||||||
// retry budget falls over to the next (resuming from what's already on disk).
|
// retry budget falls over to the next (resuming from what's already on disk).
|
||||||
var lastErr error
|
return d.overMirrors(ctx, 0, func(uri string) error {
|
||||||
for i := range d.uris {
|
return d.withRetries(ctx, uri, func() error {
|
||||||
if ctx.Err() != nil {
|
|
||||||
return ctx.Err()
|
|
||||||
}
|
|
||||||
uri := d.mirror(i)
|
|
||||||
err := d.withRetries(ctx, uri, func() error {
|
|
||||||
resumeFromDisk := !first || foreignResume
|
resumeFromDisk := !first || foreignResume
|
||||||
first = false
|
first = false
|
||||||
return d.singleOnce(ctx, out, uri, resumeFromDisk)
|
return d.singleOnce(ctx, out, uri, resumeFromDisk)
|
||||||
})
|
})
|
||||||
if err == nil {
|
})
|
||||||
return nil
|
|
||||||
}
|
|
||||||
lastErr = err
|
|
||||||
}
|
|
||||||
return lastErr
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// singleOnce performs one single-stream transfer attempt. When resumeFromDisk is
|
// singleOnce performs one single-stream transfer attempt. When resumeFromDisk is
|
||||||
@@ -1005,6 +796,19 @@ func (d *Download) singleOnce(ctx context.Context, out, uri string, resumeFromDi
|
|||||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
|
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
|
||||||
return statusError(fmt.Sprintf("%s: %s", uri, resp.Status), resp.StatusCode)
|
return statusError(fmt.Sprintf("%s: %s", uri, resp.Status), resp.StatusCode)
|
||||||
}
|
}
|
||||||
|
// On failover, resumeAt is the prefix a previous mirror wrote. If this mirror
|
||||||
|
// honours the range, make sure it serves the same file before we append to that
|
||||||
|
// prefix — a divergent total or start means different content, and writing it
|
||||||
|
// past the existing bytes would corrupt the output. Mirror the segmented guard
|
||||||
|
// (fatal, so we fall over rather than append unverified bytes). A same-mirror
|
||||||
|
// retry compares against its own earlier total, so it never false-trips.
|
||||||
|
if resp.StatusCode == http.StatusPartialContent && resumeAt > 0 {
|
||||||
|
if prev := atomic.LoadInt64(&d.total); prev > 0 {
|
||||||
|
if start, _, t, ok := parseContentRange(resp.Header.Get("Content-Range")); ok && (t != prev || start != resumeAt) {
|
||||||
|
return fatal{fmt.Errorf("%s: returned a divergent range %q (want start %d, length %d)", uri, resp.Header.Get("Content-Range"), resumeAt, prev)}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
// If we asked to resume but the server replied 200 (no range honoured), start
|
// If we asked to resume but the server replied 200 (no range honoured), start
|
||||||
// over from the top rather than appending past the existing bytes.
|
// over from the top rather than appending past the existing bytes.
|
||||||
if resp.StatusCode == http.StatusOK {
|
if resp.StatusCode == http.StatusOK {
|
||||||
@@ -1047,7 +851,7 @@ func (d *Download) singleOnce(ctx context.Context, out, uri string, resumeFromDi
|
|||||||
}
|
}
|
||||||
off += int64(rd)
|
off += int64(rd)
|
||||||
got += int64(rd)
|
got += int64(rd)
|
||||||
atomic.StoreInt64(&d.completed, off)
|
atomic.AddInt64(&d.completed, int64(rd)) // add deltas, as pump does
|
||||||
d.throttle(ctx, rd)
|
d.throttle(ctx, rd)
|
||||||
}
|
}
|
||||||
if err == io.EOF {
|
if err == io.EOF {
|
||||||
@@ -1066,13 +870,13 @@ func (d *Download) singleOnce(ctx context.Context, out, uri string, resumeFromDi
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// saveInterval is how often the .got control file is rewritten so a crash can
|
||||||
|
// resume; on a clean finish it is removed.
|
||||||
|
const saveInterval = 60 * time.Second
|
||||||
|
|
||||||
func (d *Download) saveLoop(ctx context.Context, out string, total int64, etag, lastmod string, segs []seg, done chan<- struct{}) {
|
func (d *Download) saveLoop(ctx context.Context, out string, total int64, etag, lastmod string, segs []seg, done chan<- struct{}) {
|
||||||
defer close(done)
|
defer close(done)
|
||||||
interval := d.cfg.AutoSaveInterval
|
t := time.NewTicker(saveInterval)
|
||||||
if interval <= 0 {
|
|
||||||
interval = 60 * time.Second // --auto-save-interval default
|
|
||||||
}
|
|
||||||
t := time.NewTicker(interval)
|
|
||||||
defer t.Stop()
|
defer t.Stop()
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
@@ -1103,13 +907,9 @@ func (d *Download) request(ctx context.Context, uri, rangeHdr string) (*http.Req
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
req.Header.Set("User-Agent", d.cfg.UserAgent)
|
req.Header.Set("User-Agent", userAgent)
|
||||||
if d.cfg.Referer != "" {
|
// --header sets arbitrary request headers, applied last so they can override
|
||||||
req.Header.Set("Referer", d.cfg.Referer)
|
// the User-Agent (or add a Referer, Authorization, Cookie, ...) when needed.
|
||||||
}
|
|
||||||
if d.cfg.HTTPUser != "" {
|
|
||||||
req.SetBasicAuth(d.cfg.HTTPUser, d.cfg.HTTPPasswd)
|
|
||||||
}
|
|
||||||
for _, h := range d.cfg.Headers {
|
for _, h := range d.cfg.Headers {
|
||||||
if k, v, ok := strings.Cut(h, ":"); ok {
|
if k, v, ok := strings.Cut(h, ":"); ok {
|
||||||
req.Header.Set(strings.TrimSpace(k), strings.TrimSpace(v))
|
req.Header.Set(strings.TrimSpace(k), strings.TrimSpace(v))
|
||||||
@@ -1118,33 +918,9 @@ func (d *Download) request(ctx context.Context, uri, rangeHdr string) (*http.Req
|
|||||||
if rangeHdr != "" {
|
if rangeHdr != "" {
|
||||||
req.Header.Set("Range", rangeHdr)
|
req.Header.Set("Range", rangeHdr)
|
||||||
}
|
}
|
||||||
if d.jar != nil {
|
|
||||||
d.recordURL(req.URL)
|
|
||||||
}
|
|
||||||
return req, nil
|
return req, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// recordURL remembers a contacted URL so SaveCookies can later read back the
|
|
||||||
// jar's cookies for that host. Duplicates are tolerated; saveCookieJar dedups.
|
|
||||||
func (d *Download) recordURL(u *url.URL) {
|
|
||||||
d.mu.Lock()
|
|
||||||
d.seenURLs = append(d.seenURLs, u)
|
|
||||||
d.mu.Unlock()
|
|
||||||
}
|
|
||||||
|
|
||||||
// SaveCookies writes the client's cookie jar back to the --save-cookies file,
|
|
||||||
// if one was configured. It is a no-op when cookies are not in use. main calls
|
|
||||||
// it once per HTTP download on exit (cookies are saved at session end).
|
|
||||||
func (d *Download) SaveCookies() error {
|
|
||||||
if d.cfg.SaveCookies == "" || d.jar == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
d.mu.Lock()
|
|
||||||
urls := append([]*url.URL{}, d.seenURLs...)
|
|
||||||
d.mu.Unlock()
|
|
||||||
return saveCookieJar(d.jar, urls, d.cfg.SaveCookies)
|
|
||||||
}
|
|
||||||
|
|
||||||
// seedSegmentsFromPrefix marks the first prefix bytes of the file as already
|
// seedSegmentsFromPrefix marks the first prefix bytes of the file as already
|
||||||
// downloaded, segment by segment. A foreign partial file has no per-segment
|
// downloaded, segment by segment. A foreign partial file has no per-segment
|
||||||
// metadata, so the only thing we can trust is that bytes [0,prefix) are present;
|
// metadata, so the only thing we can trust is that bytes [0,prefix) are present;
|
||||||
|
|||||||
@@ -30,25 +30,24 @@ func TestMirrorPick(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestMaxConns checks the connection ceiling min(split, mirrors*x), floored
|
// TestMaxConns checks the connection ceiling: --split segment workers, floored
|
||||||
// at 1, since that decides how many segment workers fan out.
|
// at 1. Mirrors are alternate sources, not extra connections, so they do not
|
||||||
|
// raise the count.
|
||||||
func TestMaxConns(t *testing.T) {
|
func TestMaxConns(t *testing.T) {
|
||||||
cases := []struct{ split, x, mirrors, want int }{
|
cases := []struct{ split, mirrors, want int }{
|
||||||
{5, 1, 1, 1}, // default -x1 -s5: one connection
|
{5, 1, 5}, // split is the connection count on one server
|
||||||
{5, 16, 1, 5}, // split is the ceiling on one server
|
{16, 1, 16},
|
||||||
{16, 16, 1, 16},
|
{5, 3, 5}, // 3 mirrors, still split connections
|
||||||
{5, 1, 3, 3}, // 3 mirrors * 1 per host < split
|
{0, 1, 1}, // floor at 1
|
||||||
{16, 2, 3, 6}, // 3 mirrors * 2 per host < split
|
|
||||||
{0, 1, 1, 1}, // floor at 1
|
|
||||||
}
|
}
|
||||||
for _, c := range cases {
|
for _, c := range cases {
|
||||||
uris := make([]string, c.mirrors)
|
uris := make([]string, c.mirrors)
|
||||||
for i := range uris {
|
for i := range uris {
|
||||||
uris[i] = "http://h/x"
|
uris[i] = "http://h/x"
|
||||||
}
|
}
|
||||||
d := New(uris, Config{Split: c.split, MaxConnPerServer: c.x})
|
d := New(uris, Config{Split: c.split})
|
||||||
if d.maxConns != c.want {
|
if d.maxConns != c.want {
|
||||||
t.Errorf("split=%d x=%d mirrors=%d: maxConns=%d, want %d", c.split, c.x, c.mirrors, d.maxConns, c.want)
|
t.Errorf("split=%d mirrors=%d: maxConns=%d, want %d", c.split, c.mirrors, d.maxConns, c.want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -77,8 +76,7 @@ func TestMirrorFallback(t *testing.T) {
|
|||||||
// Primary is healthy (so the probe anchors size/validators), bad is a mirror
|
// Primary is healthy (so the probe anchors size/validators), bad is a mirror
|
||||||
// that 404s every segment it's handed.
|
// that 404s every segment it's handed.
|
||||||
d := New([]string{good.URL + "/f.bin", bad.URL + "/f.bin"}, Config{
|
d := New([]string{good.URL + "/f.bin", bad.URL + "/f.bin"}, Config{
|
||||||
Dir: dir, Out: "f.bin", Split: 4, MaxConnPerServer: 2, MinSplit: 1,
|
Dir: dir, Out: "f.bin", Split: 4, MinSplit: 1, Tries: 1,
|
||||||
Tries: 1, UserAgent: "got-test",
|
|
||||||
})
|
})
|
||||||
if err := d.Run(context.Background()); err != nil {
|
if err := d.Run(context.Background()); err != nil {
|
||||||
t.Fatalf("Run with a 404 mirror should still complete via the healthy one: %v", err)
|
t.Fatalf("Run with a 404 mirror should still complete via the healthy one: %v", err)
|
||||||
@@ -121,8 +119,7 @@ func TestMirrorDivergentRejected(t *testing.T) {
|
|||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
out := filepath.Join(dir, "f.bin")
|
out := filepath.Join(dir, "f.bin")
|
||||||
d := New([]string{primary.URL + "/f.bin", div.URL + "/f.bin"}, Config{
|
d := New([]string{primary.URL + "/f.bin", div.URL + "/f.bin"}, Config{
|
||||||
Dir: dir, Out: "f.bin", Split: 4, MaxConnPerServer: 2, MinSplit: 1,
|
Dir: dir, Out: "f.bin", Split: 4, MinSplit: 1, Tries: 1,
|
||||||
Tries: 1, UserAgent: "got-test",
|
|
||||||
})
|
})
|
||||||
if err := d.Run(context.Background()); err != nil {
|
if err := d.Run(context.Background()); err != nil {
|
||||||
t.Fatalf("Run: %v", err)
|
t.Fatalf("Run: %v", err)
|
||||||
|
|||||||
@@ -5,8 +5,9 @@ import "sync/atomic"
|
|||||||
// seg is one contiguous byte range of the output file, downloaded by a single
|
// seg is one contiguous byte range of the output file, downloaded by a single
|
||||||
// ranged GET. A segment IS a byte range — a plain HTTP downloader does not need
|
// ranged GET. A segment IS a byte range — a plain HTTP downloader does not need
|
||||||
// the Piece/Segment/block layering that exists only to share code with
|
// the Piece/Segment/block layering that exists only to share code with
|
||||||
// BitTorrent. written is updated with atomic ops so Stat() can read it while a
|
// BitTorrent. start and end are fixed when the file is divided; written is
|
||||||
// worker advances it.
|
// advanced (atomically) by the segment's one worker, so Stat() and the resume
|
||||||
|
// snapshot can read its frontier while it downloads.
|
||||||
type seg struct {
|
type seg struct {
|
||||||
index int
|
index int
|
||||||
start int64 // first byte offset, inclusive
|
start int64 // first byte offset, inclusive
|
||||||
@@ -16,14 +17,16 @@ type seg struct {
|
|||||||
|
|
||||||
func (s *seg) length() int64 { return s.end - s.start + 1 }
|
func (s *seg) length() int64 { return s.end - s.start + 1 }
|
||||||
func (s *seg) done() bool { return atomic.LoadInt64(&s.written) >= s.length() }
|
func (s *seg) done() bool { return atomic.LoadInt64(&s.written) >= s.length() }
|
||||||
func (s *seg) advance(n int64) { atomic.AddInt64(&s.written, n) }
|
func (s *seg) addWritten(n int64) { atomic.AddInt64(&s.written, n) }
|
||||||
func (s *seg) progress() int64 { return atomic.LoadInt64(&s.written) }
|
func (s *seg) progress() int64 { return atomic.LoadInt64(&s.written) }
|
||||||
func (s *seg) offset() int64 { return s.start + atomic.LoadInt64(&s.written) }
|
func (s *seg) offset() int64 { return s.start + atomic.LoadInt64(&s.written) }
|
||||||
func (s *seg) remaining() int64 { return s.length() - atomic.LoadInt64(&s.written) }
|
func (s *seg) remaining() int64 { return s.length() - atomic.LoadInt64(&s.written) }
|
||||||
|
|
||||||
// makeSegments divides a file of total bytes into contiguous segments, using at
|
// makeSegments divides a file of total bytes into contiguous segments, using at
|
||||||
// most conns of them and never splitting below minSplit. The remainder lands in
|
// most conns of them and never splitting below minSplit. The remainder lands in
|
||||||
// the last segment. With conns==1 (the default) this yields one segment.
|
// the last segment. With conns==1 (the default) this yields one segment. Each
|
||||||
|
// segment gets its own worker, so the division here fixes the parallelism: there
|
||||||
|
// is no later rebalancing.
|
||||||
func makeSegments(total, minSplit int64, conns int) []seg {
|
func makeSegments(total, minSplit int64, conns int) []seg {
|
||||||
if conns < 1 {
|
if conns < 1 {
|
||||||
conns = 1
|
conns = 1
|
||||||
|
|||||||
@@ -3,6 +3,9 @@ package httpdl
|
|||||||
import (
|
import (
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -52,14 +55,14 @@ func TestSegProgress(t *testing.T) {
|
|||||||
if s.done() {
|
if s.done() {
|
||||||
t.Fatalf("new segment should not be done")
|
t.Fatalf("new segment should not be done")
|
||||||
}
|
}
|
||||||
s.advance(60)
|
s.addWritten(60)
|
||||||
if s.offset() != 160 {
|
if s.offset() != 160 {
|
||||||
t.Errorf("offset = %d, want 160", s.offset())
|
t.Errorf("offset = %d, want 160", s.offset())
|
||||||
}
|
}
|
||||||
if s.remaining() != 40 {
|
if s.remaining() != 40 {
|
||||||
t.Errorf("remaining = %d, want 40", s.remaining())
|
t.Errorf("remaining = %d, want 40", s.remaining())
|
||||||
}
|
}
|
||||||
s.advance(40)
|
s.addWritten(40)
|
||||||
if !s.done() {
|
if !s.done() {
|
||||||
t.Errorf("segment should be done after writing full length")
|
t.Errorf("segment should be done after writing full length")
|
||||||
}
|
}
|
||||||
@@ -186,3 +189,107 @@ func TestParseContentRange(t *testing.T) {
|
|||||||
t.Errorf("parseContentRange(garbage) ok = true, want false")
|
t.Errorf("parseContentRange(garbage) ok = true, want false")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// tilesCover checks that segs cover [0,total) exactly: sorted by start they must
|
||||||
|
// be contiguous with no gap and no overlap.
|
||||||
|
func tilesCover(t *testing.T, segs []seg, total int64) {
|
||||||
|
t.Helper()
|
||||||
|
sorted := append([]seg(nil), segs...)
|
||||||
|
sort.Slice(sorted, func(i, j int) bool { return sorted[i].start < sorted[j].start })
|
||||||
|
var next int64
|
||||||
|
for _, s := range sorted {
|
||||||
|
if s.start != next {
|
||||||
|
t.Fatalf("segment gap/overlap: next byte %d, got start %d", next, s.start)
|
||||||
|
}
|
||||||
|
next = s.end + 1
|
||||||
|
}
|
||||||
|
if next != total {
|
||||||
|
t.Fatalf("segments cover %d bytes, want %d", next, total)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSegmentsConcurrentCoverage drives the segments the way real workers do —
|
||||||
|
// one worker per segment, each copying its range in small chunks and committing
|
||||||
|
// with addWritten. The coverage array proves the core invariant: every byte is
|
||||||
|
// written exactly once, with no gap or overlap between adjacent segments.
|
||||||
|
func TestSegmentsConcurrentCoverage(t *testing.T) {
|
||||||
|
const (
|
||||||
|
total = int64(64 * readBuf) // 2 MiB
|
||||||
|
minSplit = int64(readBuf)
|
||||||
|
chunk = int64(readBuf / 4) // one "read"
|
||||||
|
conns = 8
|
||||||
|
)
|
||||||
|
segs := makeSegments(total, minSplit, conns)
|
||||||
|
if len(segs) < 2 {
|
||||||
|
t.Fatalf("expected several segments, got %d", len(segs))
|
||||||
|
}
|
||||||
|
cover := make([]uint32, total)
|
||||||
|
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for i := range segs {
|
||||||
|
wg.Add(1)
|
||||||
|
go func(s *seg) {
|
||||||
|
defer wg.Done()
|
||||||
|
for s.remaining() > 0 {
|
||||||
|
n := chunk
|
||||||
|
if s.remaining() < n {
|
||||||
|
n = s.remaining()
|
||||||
|
}
|
||||||
|
off := s.offset()
|
||||||
|
for j := off; j < off+n; j++ {
|
||||||
|
atomic.AddUint32(&cover[j], 1)
|
||||||
|
}
|
||||||
|
s.addWritten(n)
|
||||||
|
}
|
||||||
|
}(&segs[i])
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
for i, c := range cover {
|
||||||
|
if c != 1 {
|
||||||
|
t.Fatalf("byte %d written %d times, want exactly 1", i, c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
tilesCover(t, segs, total)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSnapshotRoundTrips checks the persistence path an interrupt-and-resume
|
||||||
|
// relies on: a snapshot of partially-written segments records each frontier, and
|
||||||
|
// segsFromControl rebuilds a set that still tiles the file and keeps the bytes
|
||||||
|
// already written.
|
||||||
|
func TestSnapshotRoundTrips(t *testing.T) {
|
||||||
|
const (
|
||||||
|
total = int64(8 * readBuf)
|
||||||
|
minSplit = int64(readBuf)
|
||||||
|
)
|
||||||
|
segs := makeSegments(total, minSplit, 4)
|
||||||
|
if len(segs) < 2 {
|
||||||
|
t.Fatalf("expected several segments, got %d", len(segs))
|
||||||
|
}
|
||||||
|
// Each segment makes some progress, so the snapshot has a real frontier to
|
||||||
|
// carry across the round trip.
|
||||||
|
var want int64
|
||||||
|
for i := range segs {
|
||||||
|
n := int64(readBuf)
|
||||||
|
if n > segs[i].length() {
|
||||||
|
n = segs[i].length()
|
||||||
|
}
|
||||||
|
segs[i].addWritten(n)
|
||||||
|
want += n
|
||||||
|
}
|
||||||
|
|
||||||
|
c := snapshot("http://x", total, "", "", segs)
|
||||||
|
if len(c.Segs) != len(segs) {
|
||||||
|
t.Fatalf("snapshot has %d segments, want %d", len(c.Segs), len(segs))
|
||||||
|
}
|
||||||
|
|
||||||
|
rebuilt := segsFromControl(&c)
|
||||||
|
tilesCover(t, rebuilt, total)
|
||||||
|
var written int64
|
||||||
|
for _, s := range rebuilt {
|
||||||
|
written += s.progress()
|
||||||
|
}
|
||||||
|
if written != want {
|
||||||
|
t.Errorf("rebuilt written = %d, want %d (bytes must survive the round trip)", written, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -45,9 +45,7 @@ func TestSingleShortBody(t *testing.T) {
|
|||||||
d := New([]string{srv.URL + "/file.bin"}, Config{
|
d := New([]string{srv.URL + "/file.bin"}, Config{
|
||||||
Dir: dir,
|
Dir: dir,
|
||||||
Out: "file.bin",
|
Out: "file.bin",
|
||||||
MaxConnPerServer: 1,
|
|
||||||
Tries: 1, // a single attempt so the retryable short read surfaces, not loops
|
Tries: 1, // a single attempt so the retryable short read surfaces, not loops
|
||||||
UserAgent: "got-test",
|
|
||||||
})
|
})
|
||||||
|
|
||||||
err := d.single(context.Background(), out)
|
err := d.single(context.Background(), out)
|
||||||
@@ -77,9 +75,7 @@ func TestSingleCompleteFullBody(t *testing.T) {
|
|||||||
d := New([]string{srv.URL + "/file.bin"}, Config{
|
d := New([]string{srv.URL + "/file.bin"}, Config{
|
||||||
Dir: dir,
|
Dir: dir,
|
||||||
Out: "file.bin",
|
Out: "file.bin",
|
||||||
MaxConnPerServer: 1,
|
|
||||||
Tries: 1,
|
Tries: 1,
|
||||||
UserAgent: "got-test",
|
|
||||||
})
|
})
|
||||||
|
|
||||||
if err := d.single(context.Background(), out); err != nil {
|
if err := d.single(context.Background(), out); err != nil {
|
||||||
|
|||||||
@@ -1,32 +0,0 @@
|
|||||||
package httpdl
|
|
||||||
|
|
||||||
import "testing"
|
|
||||||
|
|
||||||
// TestBelowSpeed covers the speedGuard decision, including the counter-reset
|
|
||||||
// case (item [1]): d.completed can jump backward when singleOnce rewrites it
|
|
||||||
// with an absolute StoreInt64 and a resumed Range is answered by a 200 that
|
|
||||||
// resets the offset to 0. A backward jump must NOT be read as a slow window, or
|
|
||||||
// a healthy download would be false-aborted.
|
|
||||||
func TestBelowSpeed(t *testing.T) {
|
|
||||||
const limit = 1000
|
|
||||||
cases := []struct {
|
|
||||||
name string
|
|
||||||
prev, now int64
|
|
||||||
want bool
|
|
||||||
}{
|
|
||||||
{"fast", 0, 5000, false},
|
|
||||||
{"exactly at limit aborts", 1000, 2000, true},
|
|
||||||
{"just under limit aborts", 1000, 1999, true},
|
|
||||||
{"just over limit ok", 1000, 2001, false},
|
|
||||||
{"stalled aborts", 4242, 4242, true},
|
|
||||||
{"counter reset is not slow", 1_000_000, 0, false},
|
|
||||||
{"small backward jump is not slow", 5000, 4500, false},
|
|
||||||
}
|
|
||||||
for _, c := range cases {
|
|
||||||
t.Run(c.name, func(t *testing.T) {
|
|
||||||
if got := belowSpeed(c.prev, c.now, limit); got != c.want {
|
|
||||||
t.Errorf("belowSpeed(%d, %d, %d) = %v; want %v", c.prev, c.now, limit, got, c.want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
263
main.go
263
main.go
@@ -17,7 +17,7 @@ import (
|
|||||||
"sort"
|
"sort"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync/atomic"
|
||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -88,30 +88,33 @@ func run(args []string) int {
|
|||||||
sessionFile := opts.Str("save-session")
|
sessionFile := opts.Str("save-session")
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
defer cancel()
|
defer cancel()
|
||||||
// --stop: cancel everything after N seconds, exactly like a Ctrl-C, so an
|
var jobsRef atomic.Pointer[[]job]
|
||||||
// unattended run can't hang forever (0 = off).
|
// publish snapshots the slice header (j is a by-value copy) and stores a
|
||||||
if stop := opts.Int("stop"); stop > 0 {
|
// pointer to it, so the signal handler can read the current jobs lock-free
|
||||||
t := time.AfterFunc(time.Duration(stop)*time.Second, cancel)
|
// without racing the `jobs = append(...)` reassignment in the follow pass.
|
||||||
defer t.Stop()
|
publish := func(j []job) { jobsRef.Store(&j) }
|
||||||
}
|
go handleSignals(cancel, sessionFile, &jobsRef)
|
||||||
jobsRef := &jobsHolder{}
|
|
||||||
go handleSignals(cancel, sessionFile, jobsRef)
|
|
||||||
|
|
||||||
eng := download.NewEngine(opts.Int("max-concurrent-downloads"))
|
eng := download.NewEngine(opts.Int("max-concurrent-downloads"))
|
||||||
|
|
||||||
uiCtx, uiCancel := context.WithCancel(context.Background())
|
uiCtx, uiCancel := context.WithCancel(context.Background())
|
||||||
|
uiDone := make(chan struct{})
|
||||||
if !opts.Bool("quiet") && !opts.Bool("dry-run") {
|
if !opts.Bool("quiet") && !opts.Bool("dry-run") {
|
||||||
rep := progress.New(eng.Snapshot, opts.Bool("human-readable"))
|
rep := progress.New(eng.Snapshot, true)
|
||||||
go rep.Run(uiCtx)
|
go func() {
|
||||||
|
rep.Run(uiCtx)
|
||||||
|
close(uiDone)
|
||||||
|
}()
|
||||||
|
} else {
|
||||||
|
close(uiDone)
|
||||||
}
|
}
|
||||||
|
|
||||||
jobsRef.set(jobs)
|
publish(jobs)
|
||||||
results := runJobs(ctx, eng, jobs)
|
results := runJobs(ctx, eng, jobs)
|
||||||
|
|
||||||
// --follow-torrent: a .torrent fetched over HTTP becomes a BitTorrent
|
// A .torrent fetched over HTTP becomes a BitTorrent download of its content,
|
||||||
// download of its content (on by default). Done as a second pass so the
|
// done as a second pass so the engine and scheduler stay simple.
|
||||||
// engine and scheduler stay simple.
|
if ctx.Err() == nil && !opts.Bool("dry-run") {
|
||||||
if opts.Bool("follow-torrent") && ctx.Err() == nil && !opts.Bool("dry-run") {
|
|
||||||
if follow := followUps(jobs, btClient, btOptions(opts)); len(follow) > 0 {
|
if follow := followUps(jobs, btClient, btOptions(opts)); len(follow) > 0 {
|
||||||
// The engine numbers each Result by its position in the slice it was
|
// The engine numbers each Result by its position in the slice it was
|
||||||
// passed, so the follow pass restarts at Index 0. Offset those by the
|
// passed, so the follow pass restarts at Index 0. Offset those by the
|
||||||
@@ -119,7 +122,7 @@ func run(args []string) int {
|
|||||||
// Index space, so the sort below restores input order deterministically.
|
// Index space, so the sort below restores input order deterministically.
|
||||||
nInitial := len(jobs)
|
nInitial := len(jobs)
|
||||||
jobs = append(jobs, follow...)
|
jobs = append(jobs, follow...)
|
||||||
jobsRef.set(jobs)
|
publish(jobs)
|
||||||
followResults := runJobs(ctx, eng, follow)
|
followResults := runJobs(ctx, eng, follow)
|
||||||
for i := range followResults {
|
for i := range followResults {
|
||||||
followResults[i].Index += nInitial
|
followResults[i].Index += nInitial
|
||||||
@@ -129,9 +132,7 @@ func run(args []string) int {
|
|||||||
}
|
}
|
||||||
|
|
||||||
uiCancel()
|
uiCancel()
|
||||||
time.Sleep(20 * time.Millisecond) // let the renderer clear its line
|
<-uiDone // wait for the renderer's final frame before printing the OK/FAIL lines
|
||||||
|
|
||||||
saveCookies(jobs)
|
|
||||||
|
|
||||||
if sessionFile != "" {
|
if sessionFile != "" {
|
||||||
if err := saveSession(sessionFile, jobs); err != nil {
|
if err := saveSession(sessionFile, jobs); err != nil {
|
||||||
@@ -147,25 +148,6 @@ func run(args []string) int {
|
|||||||
return report(results)
|
return report(results)
|
||||||
}
|
}
|
||||||
|
|
||||||
// jobsHolder lets the signal handler reach the latest job list for a
|
|
||||||
// best-effort session save on a forced (second-Ctrl-C) exit.
|
|
||||||
type jobsHolder struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
jobs []job
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *jobsHolder) set(jobs []job) {
|
|
||||||
h.mu.Lock()
|
|
||||||
h.jobs = jobs
|
|
||||||
h.mu.Unlock()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *jobsHolder) get() []job {
|
|
||||||
h.mu.Lock()
|
|
||||||
defer h.mu.Unlock()
|
|
||||||
return h.jobs
|
|
||||||
}
|
|
||||||
|
|
||||||
// job pairs a download with the source string that produced it, so we can write
|
// job pairs a download with the source string that produced it, so we can write
|
||||||
// the still-unfinished ones back to a session file.
|
// the still-unfinished ones back to a session file.
|
||||||
type job struct {
|
type job struct {
|
||||||
@@ -190,35 +172,43 @@ type failedDownload struct {
|
|||||||
err error
|
err error
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f failedDownload) Name() string { return f.name }
|
func (f *failedDownload) Name() string { return f.name }
|
||||||
func (f failedDownload) Run(context.Context) error { return f.err }
|
func (f *failedDownload) Run(context.Context) error { return f.err }
|
||||||
func (f failedDownload) Stat() download.Stat {
|
func (f *failedDownload) Stat() download.Stat {
|
||||||
return download.Stat{Name: f.name, Status: download.Errored, Total: -1}
|
return download.Stat{Name: f.name, Status: download.Errored, Total: -1}
|
||||||
}
|
}
|
||||||
|
|
||||||
// dedupeTorrents finds torrent/magnet sources that name the same torrent as an
|
// seenInfoHash records source under its torrent infohash in seen and reports
|
||||||
// earlier source (same infohash), returning each duplicate URI mapped to the
|
// whether that infohash was already recorded, returning the source that first
|
||||||
// earlier source it repeats. HTTP(S) URLs and any source whose infohash can't be
|
// claimed it. The shared client keys torrents by infohash and does not refcount,
|
||||||
// read without a network fetch are never duplicates here. The shared client keys
|
// so two sources naming one torrent would have the first to finish Drop it out
|
||||||
// torrents by infohash and does not refcount, so build() turns each duplicate
|
// from under the other; callers fail the duplicate instead. A source whose
|
||||||
// into a failed download (duplicate infohash) rather than letting two jobs
|
// infohash can't be known without a network fetch (an HTTP URL), or that is
|
||||||
// share — and Drop — one torrent.
|
// v2-only or unreadable, is never a duplicate: it isn't recorded, so it runs.
|
||||||
func dedupeTorrents(uris []string, follow bool) map[string]string {
|
func seenInfoHash(seen map[metainfo.Hash]string, source string, isFile bool) (first string, dup bool) {
|
||||||
|
h, ok := bt.SourceInfoHash(source, isFile)
|
||||||
|
if !ok {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
if first, dup = seen[h]; dup {
|
||||||
|
return first, true
|
||||||
|
}
|
||||||
|
seen[h] = source
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
|
||||||
|
// dedupeTorrents finds torrent/magnet sources that repeat an earlier source's
|
||||||
|
// infohash, returning each duplicate URI mapped to the source it repeats.
|
||||||
|
func dedupeTorrents(uris []string) map[string]string {
|
||||||
seen := map[metainfo.Hash]string{}
|
seen := map[metainfo.Hash]string{}
|
||||||
dup := map[string]string{}
|
dup := map[string]string{}
|
||||||
for _, u := range uris {
|
for _, u := range uris {
|
||||||
k := uriKind(u, follow)
|
k := uriKind(u)
|
||||||
if k != kindMagnet && k != kindTorrent {
|
if k != kindMagnet && k != kindTorrent {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
h, ok := bt.SourceInfoHash(u, k == kindTorrent)
|
if first, isDup := seenInfoHash(seen, u, k == kindTorrent); isDup {
|
||||||
if !ok {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if first, isDup := seen[h]; isDup {
|
|
||||||
dup[u] = first
|
dup[u] = first
|
||||||
} else {
|
|
||||||
seen[h] = u
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return dup
|
return dup
|
||||||
@@ -231,7 +221,7 @@ func build(opts *cli.Options, targets []target) ([]job, *torrent.Client, error)
|
|||||||
// finish would Drop it out from under the other. The duplicate is reported as
|
// finish would Drop it out from under the other. The duplicate is reported as
|
||||||
// a failed download (duplicate infohash, exit 12); the loop below does this,
|
// a failed download (duplicate infohash, exit 12); the loop below does this,
|
||||||
// so the duplicate never obtains or Drops a shared torrent.
|
// so the duplicate never obtains or Drops a shared torrent.
|
||||||
dups := dedupeTorrents(flat, opts.Bool("follow-torrent"))
|
dups := dedupeTorrents(flat)
|
||||||
|
|
||||||
overallDL := makeLimiter(opts.Size("max-overall-download-limit"))
|
overallDL := makeLimiter(opts.Size("max-overall-download-limit"))
|
||||||
overallUL := makeLimiter(opts.Size("max-overall-upload-limit"))
|
overallUL := makeLimiter(opts.Size("max-overall-upload-limit"))
|
||||||
@@ -249,7 +239,7 @@ func build(opts *cli.Options, targets []target) ([]job, *torrent.Client, error)
|
|||||||
// it (e.g. the listen port is taken) disables BitTorrent but still lets HTTP
|
// it (e.g. the listen port is taken) disables BitTorrent but still lets HTTP
|
||||||
// downloads run; those torrents then report the error.
|
// downloads run; those torrents then report the error.
|
||||||
var client *torrent.Client
|
var client *torrent.Client
|
||||||
if needsBT(opts, flat) {
|
if needsBT(flat) {
|
||||||
c, err := bt.NewClient(btClientConfig(opts, overallDL, overallUL))
|
c, err := bt.NewClient(btClientConfig(opts, overallDL, overallUL))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintln(os.Stderr, "got: bittorrent disabled:", err)
|
fmt.Fprintln(os.Stderr, "got: bittorrent disabled:", err)
|
||||||
@@ -263,14 +253,14 @@ func build(opts *cli.Options, targets []target) ([]job, *torrent.Client, error)
|
|||||||
u := t.uris[0] // primary: classifies the target and names the session source
|
u := t.uris[0] // primary: classifies the target and names the session source
|
||||||
src := strings.Join(t.uris, "\t")
|
src := strings.Join(t.uris, "\t")
|
||||||
if first, isDup := dups[u]; isDup {
|
if first, isDup := dups[u]; isDup {
|
||||||
out = append(out, job{source: src, dl: failedDownload{
|
out = append(out, job{source: src, dl: &failedDownload{
|
||||||
name: u,
|
name: u,
|
||||||
err: fmt.Errorf("%w: same torrent as %s", errDuplicate, first),
|
err: fmt.Errorf("%w: same torrent as %s", errDuplicate, first),
|
||||||
}})
|
}})
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
var dl download.Download
|
var dl download.Download
|
||||||
switch uriKind(u, opts.Bool("follow-torrent")) {
|
switch uriKind(u) {
|
||||||
case kindMagnet:
|
case kindMagnet:
|
||||||
dl = bt.New(client, u, false, bo)
|
dl = bt.New(client, u, false, bo)
|
||||||
case kindHTTP, kindFollow:
|
case kindHTTP, kindFollow:
|
||||||
@@ -289,16 +279,15 @@ func build(opts *cli.Options, targets []target) ([]job, *torrent.Client, error)
|
|||||||
}
|
}
|
||||||
|
|
||||||
// uriKind classifies a URI by scheme into the taxonomy build() and needsBT()
|
// uriKind classifies a URI by scheme into the taxonomy build() and needsBT()
|
||||||
// both switch on. Scheme wins over suffix: an http URL ending in .torrent is a
|
// both switch on. Scheme wins over suffix: an http URL ending in .torrent is
|
||||||
// file to fetch over HTTP, and with --follow-torrent it becomes a torrent only
|
// fetched over HTTP first, then followed as a torrent (kindFollow).
|
||||||
// after the fetch (kindFollow), so it still downloads over HTTP first.
|
func uriKind(u string) kind {
|
||||||
func uriKind(u string, follow bool) kind {
|
|
||||||
lu := strings.ToLower(u)
|
lu := strings.ToLower(u)
|
||||||
switch {
|
switch {
|
||||||
case strings.HasPrefix(u, "magnet:"):
|
case strings.HasPrefix(u, "magnet:"):
|
||||||
return kindMagnet
|
return kindMagnet
|
||||||
case strings.HasPrefix(u, "http://"), strings.HasPrefix(u, "https://"):
|
case strings.HasPrefix(u, "http://"), strings.HasPrefix(u, "https://"):
|
||||||
if follow && strings.HasSuffix(lu, ".torrent") {
|
if strings.HasSuffix(lu, ".torrent") {
|
||||||
return kindFollow
|
return kindFollow
|
||||||
}
|
}
|
||||||
return kindHTTP
|
return kindHTTP
|
||||||
@@ -320,10 +309,9 @@ const (
|
|||||||
|
|
||||||
// needsBT reports whether the run will involve any torrent, so we only start a
|
// needsBT reports whether the run will involve any torrent, so we only start a
|
||||||
// client when one is actually needed.
|
// client when one is actually needed.
|
||||||
func needsBT(opts *cli.Options, uris []string) bool {
|
func needsBT(uris []string) bool {
|
||||||
follow := opts.Bool("follow-torrent")
|
|
||||||
for _, u := range uris {
|
for _, u := range uris {
|
||||||
switch uriKind(u, follow) {
|
switch uriKind(u) {
|
||||||
case kindMagnet, kindTorrent, kindFollow:
|
case kindMagnet, kindTorrent, kindFollow:
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
@@ -358,6 +346,7 @@ func runJobs(ctx context.Context, eng *download.Engine, jobs []job) []download.R
|
|||||||
// so a regular download that merely ends in ".torrent" is left alone.
|
// so a regular download that merely ends in ".torrent" is left alone.
|
||||||
func followUps(jobs []job, client *torrent.Client, bo bt.Options) []job {
|
func followUps(jobs []job, client *torrent.Client, bo bt.Options) []job {
|
||||||
var out []job
|
var out []job
|
||||||
|
seen := map[metainfo.Hash]string{}
|
||||||
for _, j := range jobs {
|
for _, j := range jobs {
|
||||||
h, ok := j.dl.(*httpdl.Download)
|
h, ok := j.dl.(*httpdl.Download)
|
||||||
if !ok || j.dl.Stat().Status != download.Complete {
|
if !ok || j.dl.Stat().Status != download.Complete {
|
||||||
@@ -373,24 +362,20 @@ func followUps(jobs []job, client *torrent.Client, bo bt.Options) []job {
|
|||||||
if _, err := bt.Files(path); err != nil {
|
if _, err := bt.Files(path); err != nil {
|
||||||
continue // downloaded file is not a valid torrent
|
continue // downloaded file is not a valid torrent
|
||||||
}
|
}
|
||||||
|
// build() dedupes pass-1 sources but cannot reach a not-yet-fetched HTTP
|
||||||
|
// .torrent, so dedupe the fetched ones here too against the same hazard.
|
||||||
|
if first, dup := seenInfoHash(seen, path, true); dup {
|
||||||
|
out = append(out, job{source: path, dl: &failedDownload{
|
||||||
|
name: path,
|
||||||
|
err: fmt.Errorf("%w: same torrent as %s", errDuplicate, first),
|
||||||
|
}})
|
||||||
|
continue
|
||||||
|
}
|
||||||
out = append(out, job{source: path, dl: bt.New(client, path, true, bo)})
|
out = append(out, job{source: path, dl: bt.New(client, path, true, bo)})
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// saveCookies asks each HTTP download to write its cookie jar back to the
|
|
||||||
// --save-cookies file, persisting cookies at session end. A download with no
|
|
||||||
// --save-cookies configured is a no-op.
|
|
||||||
func saveCookies(jobs []job) {
|
|
||||||
for _, j := range jobs {
|
|
||||||
if h, ok := j.dl.(*httpdl.Download); ok {
|
|
||||||
if err := h.SaveCookies(); err != nil {
|
|
||||||
fmt.Fprintln(os.Stderr, "got:", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// target is one download: a single torrent/magnet/URL, or several HTTP mirror
|
// target is one download: a single torrent/magnet/URL, or several HTTP mirror
|
||||||
// URLs that serve identical bytes (uris[0] is the primary).
|
// URLs that serve identical bytes (uris[0] is the primary).
|
||||||
type target struct{ uris []string }
|
type target struct{ uris []string }
|
||||||
@@ -405,34 +390,30 @@ func allURIs(ts []target) []string {
|
|||||||
return u
|
return u
|
||||||
}
|
}
|
||||||
|
|
||||||
// gatherURIs collects download targets from the command line, --torrent-file,
|
// gatherURIs collects download targets from the command line and --input-file.
|
||||||
// and --input-file. Plain command-line http/https URIs are grouped into ONE
|
// Plain command-line http/https URIs are grouped into ONE mirrored download by
|
||||||
// mirrored download by default (every command-line URI names the same file);
|
// default (every command-line URI names the same file); -Z/--force-sequential
|
||||||
// -Z/--force-sequential makes each its own download. Torrents and magnets
|
// makes each its own download. Torrents, magnets, and .torrent URLs (fetched
|
||||||
// are always one target each; --input-file groups TAB-separated URIs per line.
|
// over HTTP then followed) are always one target each, since distinct torrents
|
||||||
|
// are not mirrors of one another; --input-file groups TAB-separated URIs per line.
|
||||||
func gatherURIs(opts *cli.Options, cmdURIs []string) []target {
|
func gatherURIs(opts *cli.Options, cmdURIs []string) []target {
|
||||||
var ts []target
|
var ts []target
|
||||||
follow := opts.Bool("follow-torrent")
|
|
||||||
seq := opts.Bool("force-sequential")
|
seq := opts.Bool("force-sequential")
|
||||||
var httpGroup []string
|
var httpGroup []string
|
||||||
for _, u := range cmdURIs {
|
for _, u := range cmdURIs {
|
||||||
switch uriKind(u, follow) {
|
// Only plain HTTP URLs collapse into one mirror download (alternate sources
|
||||||
case kindHTTP, kindFollow:
|
// for the same file); -Z opts out. A .torrent URL (kindFollow), like a
|
||||||
if seq {
|
// torrent or magnet, is its own download — distinct torrents are never
|
||||||
ts = append(ts, target{uris: []string{u}})
|
// mirrors of one another.
|
||||||
} else {
|
if uriKind(u) == kindHTTP && !seq {
|
||||||
httpGroup = append(httpGroup, u)
|
httpGroup = append(httpGroup, u)
|
||||||
}
|
} else {
|
||||||
default:
|
|
||||||
ts = append(ts, target{uris: []string{u}})
|
ts = append(ts, target{uris: []string{u}})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(httpGroup) > 0 {
|
if len(httpGroup) > 0 {
|
||||||
ts = append(ts, target{uris: httpGroup})
|
ts = append(ts, target{uris: httpGroup})
|
||||||
}
|
}
|
||||||
if tf := opts.Str("torrent-file"); tf != "" {
|
|
||||||
ts = append(ts, target{uris: []string{tf}})
|
|
||||||
}
|
|
||||||
if in := opts.Str("input-file"); in != "" {
|
if in := opts.Str("input-file"); in != "" {
|
||||||
ts = append(ts, readInputFile(in)...)
|
ts = append(ts, readInputFile(in)...)
|
||||||
}
|
}
|
||||||
@@ -491,8 +472,6 @@ func httpConfig(opts *cli.Options, single bool, overallDL *rate.Limiter) httpdl.
|
|||||||
Dir: opts.Str("dir"),
|
Dir: opts.Str("dir"),
|
||||||
Out: out,
|
Out: out,
|
||||||
Split: opts.Int("split"),
|
Split: opts.Int("split"),
|
||||||
MaxConnPerServer: opts.Int("max-connection-per-server"),
|
|
||||||
MinSplit: opts.Size("min-split-size"),
|
|
||||||
Tries: opts.Int("max-tries"),
|
Tries: opts.Int("max-tries"),
|
||||||
Timeout: time.Duration(opts.Int("timeout")) * time.Second,
|
Timeout: time.Duration(opts.Int("timeout")) * time.Second,
|
||||||
FileAlloc: opts.Str("file-allocation"),
|
FileAlloc: opts.Str("file-allocation"),
|
||||||
@@ -500,26 +479,9 @@ func httpConfig(opts *cli.Options, single bool, overallDL *rate.Limiter) httpdl.
|
|||||||
AllowOverwrite: opts.Bool("allow-overwrite"),
|
AllowOverwrite: opts.Bool("allow-overwrite"),
|
||||||
AutoRename: opts.Bool("auto-file-renaming"),
|
AutoRename: opts.Bool("auto-file-renaming"),
|
||||||
Headers: opts.List("header"),
|
Headers: opts.List("header"),
|
||||||
UserAgent: opts.Str("user-agent"),
|
|
||||||
Referer: opts.Str("referer"),
|
|
||||||
Proxy: opts.Str("all-proxy"),
|
|
||||||
CheckCert: opts.Bool("check-certificate"),
|
|
||||||
Limit: opts.Size("max-download-limit"),
|
|
||||||
OverallLimiter: overallDL,
|
OverallLimiter: overallDL,
|
||||||
RetryWait: time.Duration(opts.Int("retry-wait")) * time.Second,
|
|
||||||
AutoSaveInterval: time.Duration(opts.Int("auto-save-interval")) * time.Second,
|
|
||||||
ConnectTimeout: time.Duration(opts.Int("connect-timeout")) * time.Second,
|
|
||||||
LoadCookies: opts.Str("load-cookies"),
|
|
||||||
SaveCookies: opts.Str("save-cookies"),
|
|
||||||
ConditionalGet: opts.Bool("conditional-get"),
|
|
||||||
RemoteTime: opts.Bool("remote-time"),
|
|
||||||
HTTPUser: opts.Str("http-user"),
|
|
||||||
HTTPPasswd: opts.Str("http-passwd"),
|
|
||||||
LowestSpeedLimit: opts.Size("lowest-speed-limit"),
|
|
||||||
Checksum: opts.Str("checksum"),
|
Checksum: opts.Str("checksum"),
|
||||||
DryRun: opts.Bool("dry-run"),
|
DryRun: opts.Bool("dry-run"),
|
||||||
CACert: opts.Str("ca-certificate"),
|
|
||||||
DisableIPv6: opts.Bool("disable-ipv6"),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -529,13 +491,8 @@ func btClientConfig(opts *cli.Options, overallDL, overallUL *rate.Limiter) bt.Cl
|
|||||||
Dir: opts.Str("dir"),
|
Dir: opts.Str("dir"),
|
||||||
ListenPortSpec: opts.Str("listen-port"),
|
ListenPortSpec: opts.Str("listen-port"),
|
||||||
DHT: opts.Bool("enable-dht"),
|
DHT: opts.Bool("enable-dht"),
|
||||||
MaxPeers: opts.Int("bt-max-peers"),
|
|
||||||
OverallDown: overallDL,
|
OverallDown: overallDL,
|
||||||
OverallUp: overallUL,
|
OverallUp: overallUL,
|
||||||
DownLimit: opts.Size("max-download-limit"),
|
|
||||||
UpLimit: opts.Size("max-upload-limit"),
|
|
||||||
UserAgent: opts.Str("user-agent"),
|
|
||||||
DisableIPv6: opts.Bool("disable-ipv6"),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -545,8 +502,6 @@ func btOptions(opts *cli.Options) bt.Options {
|
|||||||
SeedTimeSet: opts.IsSet("seed-time"),
|
SeedTimeSet: opts.IsSet("seed-time"),
|
||||||
SeedTime: time.Duration(opts.Float("seed-time") * float64(time.Minute)),
|
SeedTime: time.Duration(opts.Float("seed-time") * float64(time.Minute)),
|
||||||
SeedRatio: opts.Float("seed-ratio"),
|
SeedRatio: opts.Float("seed-ratio"),
|
||||||
StopTimeout: time.Duration(opts.Int("bt-stop-timeout")) * time.Second,
|
|
||||||
MetaTimeout: time.Duration(opts.Int("bt-metadata-timeout")) * time.Second,
|
|
||||||
SelectFiles: parseSelect(opts.Str("select-file")),
|
SelectFiles: parseSelect(opts.Str("select-file")),
|
||||||
CheckIntegrity: opts.Bool("check-integrity"),
|
CheckIntegrity: opts.Bool("check-integrity"),
|
||||||
DryRun: opts.Bool("dry-run"),
|
DryRun: opts.Bool("dry-run"),
|
||||||
@@ -667,8 +622,6 @@ func exitCode(err error) int {
|
|||||||
return 6
|
return 6
|
||||||
case errors.Is(err, httpdl.ErrNotFound):
|
case errors.Is(err, httpdl.ErrNotFound):
|
||||||
return 3
|
return 3
|
||||||
case errors.Is(err, httpdl.ErrTooSlow):
|
|
||||||
return 5
|
|
||||||
case errors.Is(err, httpdl.ErrChecksum):
|
case errors.Is(err, httpdl.ErrChecksum):
|
||||||
return 32
|
return 32
|
||||||
case errors.Is(err, errDuplicate):
|
case errors.Is(err, errDuplicate):
|
||||||
@@ -679,42 +632,29 @@ func exitCode(err error) int {
|
|||||||
|
|
||||||
// isNameResolution reports whether err is (or wraps) a DNS lookup failure —
|
// isNameResolution reports whether err is (or wraps) a DNS lookup failure —
|
||||||
// "name resolution failed" (19), which usually points at the user's own
|
// "name resolution failed" (19), which usually points at the user's own
|
||||||
// network or resolver rather than a dead remote.
|
// network or resolver rather than a dead remote. The stdlib reports every such
|
||||||
|
// failure (no-such-host, server-misbehaving) as *net.DNSError.
|
||||||
func isNameResolution(err error) bool {
|
func isNameResolution(err error) bool {
|
||||||
var de *net.DNSError
|
var de *net.DNSError
|
||||||
if errors.As(err, &de) {
|
return errors.As(err, &de)
|
||||||
return true
|
|
||||||
}
|
|
||||||
msg := strings.ToLower(err.Error())
|
|
||||||
return strings.Contains(msg, "no such host") || strings.Contains(msg, "server misbehaving")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// isNetwork reports whether err is a connection-level failure — refused, reset,
|
// isNetwork reports whether err is a connection-level failure — refused, reset,
|
||||||
// or an unreachable host/network — a "network problem" (6).
|
// or an unreachable host/network — a "network problem" (6). A dial error from
|
||||||
|
// net/http wraps the syscall errno, so the typed check sees through it.
|
||||||
func isNetwork(err error) bool {
|
func isNetwork(err error) bool {
|
||||||
if errors.Is(err, syscall.ECONNREFUSED) || errors.Is(err, syscall.ECONNRESET) ||
|
return errors.Is(err, syscall.ECONNREFUSED) || errors.Is(err, syscall.ECONNRESET) ||
|
||||||
errors.Is(err, syscall.ENETUNREACH) || errors.Is(err, syscall.EHOSTUNREACH) {
|
errors.Is(err, syscall.ENETUNREACH) || errors.Is(err, syscall.EHOSTUNREACH)
|
||||||
return true
|
|
||||||
}
|
|
||||||
msg := strings.ToLower(err.Error())
|
|
||||||
return strings.Contains(msg, "connection refused") ||
|
|
||||||
strings.Contains(msg, "connection reset") ||
|
|
||||||
strings.Contains(msg, "network is unreachable") ||
|
|
||||||
strings.Contains(msg, "no route to host")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// isTimeout reports whether err is (or wraps) a timeout: a deadline exceeded, a
|
// isTimeout reports whether err is (or wraps) a timeout: a deadline exceeded, a
|
||||||
// net.Error that timed out, or an idle-read timeout reported as such.
|
// net.Error that timed out, or our own idle-read timeout (httpdl.ErrTimeout).
|
||||||
func isTimeout(err error) bool {
|
func isTimeout(err error) bool {
|
||||||
if errors.Is(err, context.DeadlineExceeded) {
|
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, httpdl.ErrTimeout) {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
var ne net.Error
|
var ne net.Error
|
||||||
if errors.As(err, &ne) && ne.Timeout() {
|
return errors.As(err, &ne) && ne.Timeout()
|
||||||
return true
|
|
||||||
}
|
|
||||||
msg := strings.ToLower(err.Error())
|
|
||||||
return strings.Contains(msg, "timeout") || strings.Contains(msg, "timed out")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// saveSession writes the sources of every download that did not finish to path,
|
// saveSession writes the sources of every download that did not finish to path,
|
||||||
@@ -777,14 +717,18 @@ func progressNote(s download.Stat) string {
|
|||||||
// TERM) cancels the graceful context so run() can wind down and save normally;
|
// TERM) cancels the graceful context so run() can wind down and save normally;
|
||||||
// a second forces exit, but first writes the session best-effort so the resume
|
// a second forces exit, but first writes the session best-effort so the resume
|
||||||
// list survives a force-quit.
|
// list survives a force-quit.
|
||||||
func handleSignals(cancel context.CancelFunc, sessionFile string, jobs *jobsHolder) {
|
func handleSignals(cancel context.CancelFunc, sessionFile string, jobs *atomic.Pointer[[]job]) {
|
||||||
sig := make(chan os.Signal, 2)
|
sig := make(chan os.Signal, 2)
|
||||||
signal.Notify(sig, os.Interrupt, syscall.SIGTERM)
|
signal.Notify(sig, os.Interrupt, syscall.SIGTERM)
|
||||||
<-sig // first: cancel the graceful context
|
<-sig // first: cancel the graceful context
|
||||||
cancel()
|
cancel()
|
||||||
<-sig // second: force exit
|
<-sig // second: force exit
|
||||||
if sessionFile != "" {
|
if sessionFile != "" {
|
||||||
if err := saveSession(sessionFile, jobs.get()); err != nil {
|
var js []job // nil until the first publish; saveSession handles an empty list
|
||||||
|
if p := jobs.Load(); p != nil {
|
||||||
|
js = *p
|
||||||
|
}
|
||||||
|
if err := saveSession(sessionFile, js); err != nil {
|
||||||
fmt.Fprintln(os.Stderr, "got:", err)
|
fmt.Fprintln(os.Stderr, "got:", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -799,6 +743,11 @@ func makeLimiter(bps int64) *rate.Limiter {
|
|||||||
return rate.NewLimiter(rate.Limit(bps), download.LimiterBurst(bps))
|
return rate.NewLimiter(rate.Limit(bps), download.LimiterBurst(bps))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// maxSelectIndex bounds how far a --select-file range is expanded: a torrent has
|
||||||
|
// at most a few thousand files, so a degenerate spec like "1-9999999999" must not
|
||||||
|
// spin or balloon the map. Indexes past the real file count are ignored anyway.
|
||||||
|
const maxSelectIndex = 1 << 20
|
||||||
|
|
||||||
// parseSelect parses "1,3-5" into a set of 1-based file indexes.
|
// parseSelect parses "1,3-5" into a set of 1-based file indexes.
|
||||||
func parseSelect(s string) map[int]bool {
|
func parseSelect(s string) map[int]bool {
|
||||||
s = strings.TrimSpace(s)
|
s = strings.TrimSpace(s)
|
||||||
@@ -816,10 +765,14 @@ func parseSelect(s string) map[int]bool {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
b, _ := strconv.Atoi(strings.TrimSpace(hi))
|
b, _ := strconv.Atoi(strings.TrimSpace(hi))
|
||||||
for i := a; i <= b; i++ {
|
if a < 1 {
|
||||||
if i > 0 {
|
a = 1
|
||||||
out[i] = true
|
|
||||||
}
|
}
|
||||||
|
if b > maxSelectIndex {
|
||||||
|
b = maxSelectIndex
|
||||||
|
}
|
||||||
|
for i := a; i <= b; i++ {
|
||||||
|
out[i] = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
|
|||||||
114
main_test.go
114
main_test.go
@@ -11,6 +11,9 @@ import (
|
|||||||
"syscall"
|
"syscall"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/anacrolix/torrent/bencode"
|
||||||
|
"github.com/anacrolix/torrent/metainfo"
|
||||||
|
"github.com/hanbok/got/bt"
|
||||||
"github.com/hanbok/got/cli"
|
"github.com/hanbok/got/cli"
|
||||||
"github.com/hanbok/got/download"
|
"github.com/hanbok/got/download"
|
||||||
"github.com/hanbok/got/httpdl"
|
"github.com/hanbok/got/httpdl"
|
||||||
@@ -66,18 +69,15 @@ func TestExitCode(t *testing.T) {
|
|||||||
err error
|
err error
|
||||||
want int
|
want int
|
||||||
}{
|
}{
|
||||||
{"idle timeout", errors.New("idle timeout: no data received"), 2},
|
{"idle timeout", fmt.Errorf("segment 0: %w (after 1m0s)", httpdl.ErrTimeout), 2},
|
||||||
{"metadata timed out", errors.New("timed out fetching metadata after 3s"), 2},
|
{"metadata timed out", fmt.Errorf("timed out fetching metadata after 3s: %w", context.DeadlineExceeded), 2},
|
||||||
{"deadline", context.DeadlineExceeded, 2},
|
{"deadline", context.DeadlineExceeded, 2},
|
||||||
{"file exists", fmt.Errorf("/tmp/x: %w (use --allow-overwrite or -c)", httpdl.ErrOutputExists), 13},
|
{"file exists", fmt.Errorf("/tmp/x: %w (use --allow-overwrite or -c)", httpdl.ErrOutputExists), 13},
|
||||||
{"dns typed", dnsErr, 19},
|
{"dns typed", dnsErr, 19},
|
||||||
{"dns wrapped through retries", fmt.Errorf("http://x: %w (after 5 tries)", dnsErr), 19},
|
{"dns wrapped through retries", fmt.Errorf("http://x: %w (after 5 tries)", dnsErr), 19},
|
||||||
{"dns by message", errors.New("dial tcp: lookup nope.invalid: no such host"), 19},
|
|
||||||
{"refused typed", refused, 6},
|
{"refused typed", refused, 6},
|
||||||
{"refused wrapped through retries", fmt.Errorf("http://x: %w (after 5 tries)", refused), 6},
|
{"refused wrapped through retries", fmt.Errorf("http://x: %w (after 5 tries)", refused), 6},
|
||||||
{"refused by message", errors.New("dial tcp 127.0.0.1:1: connect: connection refused"), 6},
|
|
||||||
{"not found", fmt.Errorf("%w: http://x/y: 404 Not Found", httpdl.ErrNotFound), 3},
|
{"not found", fmt.Errorf("%w: http://x/y: 404 Not Found", httpdl.ErrNotFound), 3},
|
||||||
{"too slow", fmt.Errorf("http://x: %w (<= %d bytes/sec)", httpdl.ErrTooSlow, 1024), 5},
|
|
||||||
{"checksum", fmt.Errorf("/tmp/x: %w (want a, got b)", httpdl.ErrChecksum), 32},
|
{"checksum", fmt.Errorf("/tmp/x: %w (want a, got b)", httpdl.ErrChecksum), 32},
|
||||||
{"duplicate", fmt.Errorf("%w: same torrent as y", errDuplicate), 12},
|
{"duplicate", fmt.Errorf("%w: same torrent as y", errDuplicate), 12},
|
||||||
{"unknown", errors.New("disk full"), 1},
|
{"unknown", errors.New("disk full"), 1},
|
||||||
@@ -103,7 +103,7 @@ func TestDedupeTorrents(t *testing.T) {
|
|||||||
c := "magnet:?xt=urn:btih:1111111111111111111111111111111111111111&dn=other"
|
c := "magnet:?xt=urn:btih:1111111111111111111111111111111111111111&dn=other"
|
||||||
web := "http://example.com/file.iso"
|
web := "http://example.com/file.iso"
|
||||||
|
|
||||||
dup := dedupeTorrents([]string{a, web, b, c}, true)
|
dup := dedupeTorrents([]string{a, web, b, c})
|
||||||
|
|
||||||
// b is the only duplicate (of a); a, web and c are not duplicates.
|
// b is the only duplicate (of a); a, web and c are not duplicates.
|
||||||
if len(dup) != 1 || dup[b] != a {
|
if len(dup) != 1 || dup[b] != a {
|
||||||
@@ -112,8 +112,8 @@ func TestDedupeTorrents(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TestGatherURIsGrouping checks the command-line grouping: plain http URIs
|
// TestGatherURIsGrouping checks the command-line grouping: plain http URIs
|
||||||
// become one mirrored download by default, -Z splits them, and torrents/magnets
|
// become one mirrored download by default, -Z splits them, and torrents,
|
||||||
// stay separate.
|
// magnets, and .torrent URLs stay separate.
|
||||||
func TestGatherURIsGrouping(t *testing.T) {
|
func TestGatherURIsGrouping(t *testing.T) {
|
||||||
parse := func(args ...string) []target {
|
parse := func(args ...string) []target {
|
||||||
res, err := cli.Parse(append([]string{"--no-conf"}, args...))
|
res, err := cli.Parse(append([]string{"--no-conf"}, args...))
|
||||||
@@ -131,6 +131,43 @@ func TestGatherURIsGrouping(t *testing.T) {
|
|||||||
if ts := parse("http://a/x", "magnet:?xt=urn:btih:abc"); len(ts) != 2 {
|
if ts := parse("http://a/x", "magnet:?xt=urn:btih:abc"); len(ts) != 2 {
|
||||||
t.Errorf("http + magnet: got %d targets, want 2 (magnet never mirrors)", len(ts))
|
t.Errorf("http + magnet: got %d targets, want 2 (magnet never mirrors)", len(ts))
|
||||||
}
|
}
|
||||||
|
// Distinct .torrent URLs are separate downloads, not mirrors of one file
|
||||||
|
// (follow-torrent defaults on, so each is fetched then followed).
|
||||||
|
if ts := parse("http://a/x.torrent", "http://b/y.torrent", "http://c/z.torrent"); len(ts) != 3 {
|
||||||
|
t.Errorf(".torrent URLs: got %d targets, want 3 separate downloads", len(ts))
|
||||||
|
}
|
||||||
|
// A plain http URL still mirror-groups even alongside a .torrent URL: one
|
||||||
|
// http group of 1 + one torrent target = 2.
|
||||||
|
if ts := parse("http://a/x", "http://b/y.torrent"); len(ts) != 2 {
|
||||||
|
t.Errorf("http + .torrent: got %d targets, want 2", len(ts))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestHTTPConnDefault locks the parallelism default: a bare `got URL` must use
|
||||||
|
// several connections, not one. --split is the single speed dial — it is the
|
||||||
|
// connection count directly, defaulting to 5.
|
||||||
|
func TestHTTPConnDefault(t *testing.T) {
|
||||||
|
cfg := func(args ...string) httpdl.Config {
|
||||||
|
res, err := cli.Parse(append([]string{"--no-conf"}, args...))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse %v: %v", args, err)
|
||||||
|
}
|
||||||
|
return httpConfig(res.Options, true, nil)
|
||||||
|
}
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
args []string
|
||||||
|
want int
|
||||||
|
}{
|
||||||
|
{"bare URL uses split default", []string{"http://a/x"}, 5},
|
||||||
|
{"-s16 means 16", []string{"-s16", "http://a/x"}, 16},
|
||||||
|
{"-s1 stays single", []string{"-s1", "http://a/x"}, 1},
|
||||||
|
}
|
||||||
|
for _, tc := range tests {
|
||||||
|
if c := cfg(tc.args...); c.Split != tc.want {
|
||||||
|
t.Errorf("%s: Split = %d, want %d", tc.name, c.Split, tc.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestReadInputFileTAB checks the input-file grouping: TAB-separated URIs on
|
// TestReadInputFileTAB checks the input-file grouping: TAB-separated URIs on
|
||||||
@@ -152,10 +189,69 @@ func TestReadInputFileTAB(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// writeTorrent creates a .torrent file at dir/name whose info dict is built from
|
||||||
|
// a file of the given content and internal name. Two calls with the same
|
||||||
|
// internalName and content produce the same infohash regardless of the output
|
||||||
|
// filename, so the test can make two .torrent files that name one torrent.
|
||||||
|
func writeTorrent(t *testing.T, dir, name, internalName string, content []byte) string {
|
||||||
|
t.Helper()
|
||||||
|
data := filepath.Join(dir, internalName)
|
||||||
|
if err := os.WriteFile(data, content, 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
info := metainfo.Info{PieceLength: 32 * 1024}
|
||||||
|
if err := info.BuildFromFilePath(data); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
b, err := bencode.Marshal(info)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
tp := filepath.Join(dir, name)
|
||||||
|
f, err := os.Create(tp)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
if err := (&metainfo.MetaInfo{InfoBytes: b}).Write(f); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return tp
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestFollowUpInfoHashDedup covers the fix for two fetched .torrent files that
|
||||||
|
// name the same torrent: the second must be flagged a duplicate so followUps
|
||||||
|
// fails it rather than letting both share — and Drop — one torrent on the client.
|
||||||
|
func TestFollowUpInfoHashDedup(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
a := writeTorrent(t, dir, "a.torrent", "same", []byte("AAAAAAAA"))
|
||||||
|
b := writeTorrent(t, dir, "b.torrent", "same", []byte("AAAAAAAA")) // same infohash as a
|
||||||
|
c := writeTorrent(t, dir, "c.torrent", "diff", []byte("BBBBBBBB")) // a distinct torrent
|
||||||
|
|
||||||
|
// Sanity-check the fixture: a and b really share an infohash, c differs.
|
||||||
|
ha, _ := bt.SourceInfoHash(a, true)
|
||||||
|
hb, _ := bt.SourceInfoHash(b, true)
|
||||||
|
hc, _ := bt.SourceInfoHash(c, true)
|
||||||
|
if ha != hb || ha == hc {
|
||||||
|
t.Fatalf("fixture: want a==b!=c infohashes, got a=%v b=%v c=%v", ha, hb, hc)
|
||||||
|
}
|
||||||
|
|
||||||
|
seen := map[metainfo.Hash]string{}
|
||||||
|
if first, dup := seenInfoHash(seen, a, true); dup {
|
||||||
|
t.Fatalf("a is the first occurrence, wrongly flagged a duplicate of %s", first)
|
||||||
|
}
|
||||||
|
if first, dup := seenInfoHash(seen, b, true); !dup || first != a {
|
||||||
|
t.Errorf("b should duplicate a: got first=%q dup=%v, want %q true", first, dup, a)
|
||||||
|
}
|
||||||
|
if _, dup := seenInfoHash(seen, c, true); dup {
|
||||||
|
t.Errorf("c is a distinct torrent, wrongly flagged a duplicate")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestReportExit(t *testing.T) {
|
func TestReportExit(t *testing.T) {
|
||||||
canceled := download.Result{Name: "a", Err: context.Canceled}
|
canceled := download.Result{Name: "a", Err: context.Canceled}
|
||||||
done := download.Result{Name: "b", Err: nil}
|
done := download.Result{Name: "b", Err: nil}
|
||||||
timeout := download.Result{Name: "c", Err: errors.New("idle timeout")}
|
timeout := download.Result{Name: "c", Err: httpdl.ErrTimeout}
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
|
|||||||
@@ -41,12 +41,6 @@ func humanSize(n int64, human bool) string {
|
|||||||
return fmt.Sprintf("%.0f%ciB", val, suffix)
|
return fmt.Sprintf("%.0f%ciB", val, suffix)
|
||||||
}
|
}
|
||||||
|
|
||||||
// speed formats a bytes-per-second rate. The DL:/UL: fields show the bare
|
|
||||||
// abbreviated size with no "/s" suffix.
|
|
||||||
func speed(bytesPerSec int64, human bool) string {
|
|
||||||
return humanSize(bytesPerSec, human)
|
|
||||||
}
|
|
||||||
|
|
||||||
// secfmt renders a duration as "1h2m3s", appending each unit only when it is
|
// secfmt renders a duration as "1h2m3s", appending each unit only when it is
|
||||||
// nonzero (but still showing seconds when the whole input is 0):
|
// nonzero (but still showing seconds when the whole input is 0):
|
||||||
// 3600s->"1h", 120s->"2m", 3720s->"1h2m". A non-positive or absurd
|
// 3600s->"1h", 120s->"2m", 3720s->"1h2m". A non-positive or absurd
|
||||||
|
|||||||
@@ -53,16 +53,6 @@ func TestSecfmt(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSpeedNoSuffix(t *testing.T) {
|
|
||||||
// The DL:/UL: fields show the bare abbreviated size with no "/s".
|
|
||||||
if got := speed(1536, true); got != "1.5KiB" {
|
|
||||||
t.Errorf("speed(1536,true) = %q, want %q", got, "1.5KiB")
|
|
||||||
}
|
|
||||||
if got := speed(2048, false); got != "2048B" {
|
|
||||||
t.Errorf("speed(2048,false) = %q, want %q", got, "2048B")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPercent(t *testing.T) {
|
func TestPercent(t *testing.T) {
|
||||||
if got := percent(50, 100); got != 50 {
|
if got := percent(50, 100); got != 50 {
|
||||||
t.Errorf("percent(50,100) = %d, want 50", got)
|
t.Errorf("percent(50,100) = %d, want 50", got)
|
||||||
|
|||||||
@@ -1,7 +1,8 @@
|
|||||||
// Package progress renders a live, single-line status display. One goroutine ticks
|
// Package progress renders a live status display: a summary line over one
|
||||||
// once a second, pulls a snapshot of the running downloads, and derives speeds
|
// aligned row per download, collapsing to the summary alone when the rows would
|
||||||
// from the change since the previous tick. There are no locks here: the data
|
// not fit. One goroutine ticks once a second, pulls a snapshot of the running
|
||||||
// arrives by value through the snapshot function.
|
// downloads, and derives speeds from the change since the previous tick. There
|
||||||
|
// are no locks here: the data arrives by value through the snapshot function.
|
||||||
package progress
|
package progress
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -9,6 +10,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
"unicode/utf8"
|
"unicode/utf8"
|
||||||
@@ -25,8 +27,10 @@ type Reporter struct {
|
|||||||
w io.Writer
|
w io.Writer
|
||||||
tty bool
|
tty bool
|
||||||
width int
|
width int
|
||||||
|
height int
|
||||||
|
|
||||||
win map[string]*speedWindow // keyed by Stat.ID
|
win map[string]*speedWindow // keyed by Stat.ID
|
||||||
|
prevLines int // lines drawn in the previous TTY frame, redrawn in place
|
||||||
lastLog time.Time // last non-TTY line emitted
|
lastLog time.Time // last non-TTY line emitted
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -62,6 +66,7 @@ func New(snap func() []download.Stat, human bool) *Reporter {
|
|||||||
w: os.Stdout,
|
w: os.Stdout,
|
||||||
tty: term.IsTerminal(int(os.Stdout.Fd())),
|
tty: term.IsTerminal(int(os.Stdout.Fd())),
|
||||||
width: 80,
|
width: 80,
|
||||||
|
height: 24,
|
||||||
win: map[string]*speedWindow{},
|
win: map[string]*speedWindow{},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -103,89 +108,251 @@ func (r *Reporter) render(final bool) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
var line string
|
|
||||||
switch len(stats) {
|
|
||||||
case 0:
|
|
||||||
if !final && r.tty {
|
|
||||||
return // nothing active; leave the line as-is
|
|
||||||
}
|
|
||||||
case 1:
|
|
||||||
line = r.lineOne(stats[0])
|
|
||||||
default:
|
|
||||||
line = r.lineMany(stats)
|
|
||||||
}
|
|
||||||
|
|
||||||
if r.tty {
|
if r.tty {
|
||||||
r.width = termWidth(r.width)
|
r.width = termWidth(r.width)
|
||||||
line = clampRunes(line, r.width)
|
r.height = termHeight(r.height)
|
||||||
fmt.Fprint(r.w, "\r"+line+"\x1b[K")
|
r.redraw(r.block(stats), final)
|
||||||
if final {
|
|
||||||
fmt.Fprintln(r.w)
|
|
||||||
}
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
// Not a TTY: can't redraw one line, so print sparingly (and always the
|
// Not a TTY: no cursor control, so emit the summary line sparingly (and always
|
||||||
// final line) to keep redirected output and logs readable.
|
// the final one) to keep redirected output and logs readable.
|
||||||
|
var line string
|
||||||
|
if len(stats) > 0 {
|
||||||
|
line = r.summary(stats)
|
||||||
|
}
|
||||||
if line != "" && (final || now.Sub(r.lastLog) >= logEvery) {
|
if line != "" && (final || now.Sub(r.lastLog) >= logEvery) {
|
||||||
fmt.Fprintln(r.w, line)
|
fmt.Fprintln(r.w, line)
|
||||||
r.lastLog = now
|
r.lastLog = now
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Reporter) lineOne(s download.Stat) string {
|
// nameW is the fixed rune width of the name column in the table, matching
|
||||||
dl, ul := r.rates(s)
|
// elide's cap so a padded name lines the columns up.
|
||||||
seeding := s.Status == download.Seeding
|
const nameW = 20
|
||||||
var b strings.Builder
|
|
||||||
fmt.Fprintf(&b, "[%s ", shortName(s.Name))
|
// block builds the lines of the live display: a summary line, the column header,
|
||||||
// While seeding, report the share ratio in place of the size block and drop
|
// and one row per download — or just the one-line summary when the table would
|
||||||
// the DL: field; otherwise show completed/total (or bare completed).
|
// not fit the window. A single download renders the same way as several.
|
||||||
if seeding {
|
func (r *Reporter) block(stats []download.Stat) []string {
|
||||||
fmt.Fprintf(&b, "SEED(%.1f)", ratio(s.Uploaded, s.Completed))
|
if len(stats) == 0 {
|
||||||
} else if s.Total > 0 {
|
return nil
|
||||||
fmt.Fprintf(&b, "%s/%s(%d%%)", humanSize(s.Completed, r.human), humanSize(s.Total, r.human), percent(s.Completed, s.Total))
|
|
||||||
} else {
|
|
||||||
b.WriteString(humanSize(s.Completed, r.human))
|
|
||||||
}
|
}
|
||||||
// CN is always shown (including HTTP); SD is shown for any torrent.
|
if len(stats)+3 > r.height {
|
||||||
fmt.Fprintf(&b, " CN:%d", s.Conns)
|
// The block is repainted in place by moving the cursor up over it, which
|
||||||
if s.IsBT {
|
// breaks if printing the last row reaches the screen's bottom row and
|
||||||
fmt.Fprintf(&b, " SD:%d", s.Seeders)
|
// scrolls the frame out from under the cursor. term height counts the row
|
||||||
|
// the shell prompt occupies, so a summary + header + one row each
|
||||||
|
// (len+2 lines) must still leave the bottom row free: fall back to the
|
||||||
|
// single summary line once len+2 would fill to the bottom.
|
||||||
|
return []string{r.summary(stats)}
|
||||||
}
|
}
|
||||||
if !seeding {
|
out := make([]string, 0, len(stats)+2)
|
||||||
fmt.Fprintf(&b, " DL:%s", speed(dl, r.human))
|
out = append(out, r.summary(stats), tableHeader())
|
||||||
|
for _, s := range stats {
|
||||||
|
out = append(out, r.tableRow(s))
|
||||||
}
|
}
|
||||||
if seeding || ul > 0 {
|
return out
|
||||||
fmt.Fprintf(&b, " UL:%s", speed(ul, r.human))
|
|
||||||
}
|
|
||||||
if !seeding && s.Total > 0 && dl > 0 {
|
|
||||||
eta := time.Duration(float64(s.Total-s.Completed)/float64(dl)) * time.Second
|
|
||||||
fmt.Fprintf(&b, " ETA:%s", secfmt(eta))
|
|
||||||
}
|
|
||||||
b.WriteByte(']')
|
|
||||||
return b.String()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Reporter) lineMany(stats []download.Stat) string {
|
// redraw repaints the block in place: move to the top of the previous frame,
|
||||||
|
// clear it, and print the new lines clamped to the terminal width. A stray line
|
||||||
|
// from stderr that lands in the block is painted over on the next tick.
|
||||||
|
func (r *Reporter) redraw(lines []string, final bool) {
|
||||||
|
if r.prevLines > 0 {
|
||||||
|
if r.prevLines > 1 {
|
||||||
|
fmt.Fprintf(r.w, "\x1b[%dA", r.prevLines-1) // up to the first line
|
||||||
|
}
|
||||||
|
fmt.Fprint(r.w, "\r\x1b[J") // column 0, clear to end of screen
|
||||||
|
}
|
||||||
|
for i, ln := range lines {
|
||||||
|
if i > 0 {
|
||||||
|
fmt.Fprint(r.w, "\n")
|
||||||
|
}
|
||||||
|
fmt.Fprint(r.w, clampRunes(ln, r.width))
|
||||||
|
}
|
||||||
|
r.prevLines = len(lines)
|
||||||
|
if final && len(lines) > 0 {
|
||||||
|
fmt.Fprintln(r.w) // leave the cursor below the block for the OK/FAIL lines
|
||||||
|
r.prevLines = 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// summary is the header/aggregate line: a count of downloads in each state, the
|
||||||
|
// total down/up speed, and a stalled count when any download has gone quiet. The
|
||||||
|
// state breakdown replaces a single "active" tally so seeding and queued
|
||||||
|
// downloads are not lumped in with the ones actually moving bytes.
|
||||||
|
func (r *Reporter) summary(stats []download.Stat) string {
|
||||||
|
if len(stats) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
var totDL, totUL int64
|
var totDL, totUL int64
|
||||||
|
var downloading, seeding, queued, stalled int
|
||||||
for _, s := range stats {
|
for _, s := range stats {
|
||||||
dl, ul := r.rates(s)
|
dl, ul := r.rates(s)
|
||||||
totDL += dl
|
totDL += dl
|
||||||
totUL += ul
|
totUL += ul
|
||||||
}
|
switch s.Status {
|
||||||
var b strings.Builder
|
case download.Waiting:
|
||||||
fmt.Fprintf(&b, "[%d active DL:%s UL:%s]", len(stats), speed(totDL, r.human), speed(totUL, r.human))
|
queued++
|
||||||
for i, s := range stats {
|
case download.Seeding:
|
||||||
if i >= 4 {
|
seeding++
|
||||||
fmt.Fprintf(&b, "[+%d]", len(stats)-i)
|
default:
|
||||||
break
|
downloading++
|
||||||
}
|
if r.isStalled(s, dl) {
|
||||||
if s.Total > 0 {
|
stalled++
|
||||||
fmt.Fprintf(&b, "[%s %d%%]", shortName(s.Name), percent(s.Completed, s.Total))
|
|
||||||
} else {
|
|
||||||
fmt.Fprintf(&b, "[%s %s]", shortName(s.Name), humanSize(s.Completed, r.human))
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return b.String()
|
}
|
||||||
|
parts := []string{fmt.Sprintf("%d downloading", downloading)}
|
||||||
|
if seeding > 0 {
|
||||||
|
parts = append(parts, fmt.Sprintf("%d seeding", seeding))
|
||||||
|
}
|
||||||
|
if queued > 0 {
|
||||||
|
parts = append(parts, fmt.Sprintf("%d queued", queued))
|
||||||
|
}
|
||||||
|
line := strings.Join(parts, " · ") + fmt.Sprintf(" DL %s UL %s", r.rateStr(totDL), r.rateStr(totUL))
|
||||||
|
if stalled > 0 {
|
||||||
|
line += fmt.Sprintf(" (%d stalled)", stalled)
|
||||||
|
}
|
||||||
|
return line
|
||||||
|
}
|
||||||
|
|
||||||
|
// tableHeader names the columns once so the rows can drop the per-field DL:/UL:
|
||||||
|
// labels. It uses tableRow's exact format string so the headings line up over
|
||||||
|
// their columns. SIZE is completed/total; CN is the connection/peer count and SD
|
||||||
|
// the connected seeders — the fields that tell a stalled download (no peers)
|
||||||
|
// from a slow one (peers, no speed).
|
||||||
|
func tableHeader() string {
|
||||||
|
return fmt.Sprintf("%s %4s %-13s %-8s %3s %3s %s", padRune("NAME", nameW), "%", "SIZE", "RATE", "CN", "SD", "ETA")
|
||||||
|
}
|
||||||
|
|
||||||
|
// tableRow renders one download as aligned columns under tableHeader:
|
||||||
|
// "name pct size rate cn sd tail". The name is middle-elided to keep its
|
||||||
|
// distinguishing tail; percent, rate and the peer counts are fixed-width so the
|
||||||
|
// digits scan straight down; size is completed/total; the rate column carries
|
||||||
|
// the download speed or a state word (seeding/queued/...), and the tail carries
|
||||||
|
// the ETA, the share ratio while seeding, or a stalled flag.
|
||||||
|
func (r *Reporter) tableRow(s download.Stat) string {
|
||||||
|
dl, ul := r.rates(s)
|
||||||
|
rate, tail := r.rowMetric(s, dl, ul)
|
||||||
|
cn, sd := peerFields(s)
|
||||||
|
row := fmt.Sprintf("%s %4s %-13s %-8s %3s %3s %s", padRune(elide(s.Name, nameW), nameW), pctField(s), sizeField(s, r.human), rate, cn, sd, tail)
|
||||||
|
// A queued row's trailing SIZE/CN/SD/ETA columns are blank; drop the trailing
|
||||||
|
// spaces so the line ends where its content does.
|
||||||
|
return strings.TrimRight(row, " ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// sizeField is the absolute-progress column: completed/total (e.g.
|
||||||
|
// "950MiB/5.0GiB"), just the completed bytes when the total is unknown (an HTTP
|
||||||
|
// stream with no Content-Length, or a pre-metadata magnet), or blank for a
|
||||||
|
// download that has not started.
|
||||||
|
func sizeField(s download.Stat, human bool) string {
|
||||||
|
switch {
|
||||||
|
case s.Status == download.Waiting:
|
||||||
|
return ""
|
||||||
|
case s.Total > 0:
|
||||||
|
return humanSize(s.Completed, human) + "/" + humanSize(s.Total, human)
|
||||||
|
case s.Completed > 0:
|
||||||
|
return humanSize(s.Completed, human)
|
||||||
|
default:
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// peerFields formats the connection (CN) and seeder (SD) columns. SD is blank
|
||||||
|
// for HTTP, which has no seeders, and both are blank for a download with no live
|
||||||
|
// connections (queued, done, failed) so the columns aren't noise where they
|
||||||
|
// carry no meaning.
|
||||||
|
func peerFields(s download.Stat) (cn, sd string) {
|
||||||
|
switch s.Status {
|
||||||
|
case download.Active, download.Seeding:
|
||||||
|
cn = strconv.Itoa(s.Conns)
|
||||||
|
if s.IsBT {
|
||||||
|
sd = strconv.Itoa(s.Seeders)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return cn, sd
|
||||||
|
}
|
||||||
|
|
||||||
|
// rowMetric returns a table row's rate column and its tail for the download's
|
||||||
|
// state: the rate is the download speed or the state word (seeding/queued/done/
|
||||||
|
// failed); the tail is an ETA, the share ratio while seeding (the seed-stop
|
||||||
|
// criterion, more telling than an idle upload rate), or a "stalled" flag.
|
||||||
|
func (r *Reporter) rowMetric(s download.Stat, dl, ul int64) (rate, tail string) {
|
||||||
|
switch s.Status {
|
||||||
|
case download.Waiting:
|
||||||
|
return "queued", ""
|
||||||
|
case download.Seeding:
|
||||||
|
return "seeding", fmt.Sprintf("r%.2f", ratio(s.Uploaded, s.Completed))
|
||||||
|
case download.Complete:
|
||||||
|
return "done", ""
|
||||||
|
case download.Errored:
|
||||||
|
return "failed", ""
|
||||||
|
default:
|
||||||
|
rate = r.rateStr(dl)
|
||||||
|
switch {
|
||||||
|
case dl > 0 && s.Total > 0:
|
||||||
|
tail = secfmt(etaDuration(s.Total-s.Completed, dl))
|
||||||
|
case r.isStalled(s, dl):
|
||||||
|
tail = "stalled"
|
||||||
|
}
|
||||||
|
return rate, tail
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// rateStr formats a bytes/sec speed for display, e.g. "388KiB/s".
|
||||||
|
func (r *Reporter) rateStr(bps int64) string {
|
||||||
|
return humanSize(bps, r.human) + "/s"
|
||||||
|
}
|
||||||
|
|
||||||
|
// maxETASeconds caps an ETA before it becomes a Duration: a near-stalled download
|
||||||
|
// yields an astronomically large seconds value, and time.Duration(secs) *
|
||||||
|
// time.Second would overflow int64 and wrap to a bogus tiny ETA. secfmt renders
|
||||||
|
// anything past 99h as "--", so clamping to 100h loses nothing.
|
||||||
|
const maxETASeconds = 100 * 3600
|
||||||
|
|
||||||
|
func etaDuration(remaining, dl int64) time.Duration {
|
||||||
|
secs := float64(remaining) / float64(dl)
|
||||||
|
if secs > maxETASeconds {
|
||||||
|
secs = maxETASeconds
|
||||||
|
}
|
||||||
|
return time.Duration(secs) * time.Second
|
||||||
|
}
|
||||||
|
|
||||||
|
// isStalled reports an active download that has gone a while with no new bytes,
|
||||||
|
// so the table can flag it rather than leave the eye to read DL:0B. A download
|
||||||
|
// too young to have a rate yet is not stalled.
|
||||||
|
func (r *Reporter) isStalled(s download.Stat, dl int64) bool {
|
||||||
|
if s.Status != download.Active || dl > 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
w := r.win[s.ID]
|
||||||
|
if w == nil || len(w.samples) < 2 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return w.samples[len(w.samples)-1].t.Sub(w.samples[0].t) >= 4*time.Second
|
||||||
|
}
|
||||||
|
|
||||||
|
// pctField is the percent column: a number, full for seeding/complete, or "--"
|
||||||
|
// when the total is unknown.
|
||||||
|
func pctField(s download.Stat) string {
|
||||||
|
switch {
|
||||||
|
case s.Status == download.Seeding || s.Status == download.Complete:
|
||||||
|
return "100%"
|
||||||
|
case s.Total > 0:
|
||||||
|
return fmt.Sprintf("%3d%%", percent(s.Completed, s.Total))
|
||||||
|
default:
|
||||||
|
return " --"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// padRune right-pads s with spaces to w runes (it is never longer, having been
|
||||||
|
// through shortName) so table columns line up on rune count.
|
||||||
|
func padRune(s string, w int) string {
|
||||||
|
if n := utf8.RuneCountInString(s); n < w {
|
||||||
|
return s + strings.Repeat(" ", w-n)
|
||||||
|
}
|
||||||
|
return s
|
||||||
}
|
}
|
||||||
|
|
||||||
// add records a sample and drops any that have fallen out of the window. A
|
// add records a sample and drops any that have fallen out of the window. A
|
||||||
@@ -244,15 +411,25 @@ func ratio(uploaded, completed int64) float64 {
|
|||||||
return float64(uploaded) / float64(completed)
|
return float64(uploaded) / float64(completed)
|
||||||
}
|
}
|
||||||
|
|
||||||
func shortName(s string) string {
|
// elide shortens s to at most w runes, keeping the head and the tail joined by a
|
||||||
const max = 20
|
// single "…". Download names often share a long prefix (a release group, a
|
||||||
// Truncate on runes, not bytes, so a multibyte name (CJK, emoji) is never
|
// series) and differ only at the end (an episode or volume number), so dropping
|
||||||
// cut mid-rune.
|
// the middle keeps what tells two downloads apart — where a leading-only
|
||||||
if utf8.RuneCountInString(s) > max {
|
// truncation would collapse them to the same string. A name within w prints
|
||||||
return string([]rune(s)[:max-1]) + "~"
|
// whole. It works on runes, never bytes, so a multibyte glyph (CJK, emoji) is
|
||||||
}
|
// never split.
|
||||||
|
func elide(s string, w int) string {
|
||||||
|
r := []rune(s)
|
||||||
|
if len(r) <= w {
|
||||||
return s
|
return s
|
||||||
}
|
}
|
||||||
|
if w <= 1 {
|
||||||
|
return string(r[:w])
|
||||||
|
}
|
||||||
|
head := w / 2 // the w-1 kept runes split around one "…",
|
||||||
|
tail := w - 1 - head // head taking the extra rune when w is even
|
||||||
|
return string(r[:head]) + "…" + string(r[len(r)-tail:])
|
||||||
|
}
|
||||||
|
|
||||||
// clampRunes limits a line to width runes (not bytes) so multibyte glyphs are
|
// clampRunes limits a line to width runes (not bytes) so multibyte glyphs are
|
||||||
// never split when the terminal is narrow. A non-positive width leaves it whole.
|
// never split when the terminal is narrow. A non-positive width leaves it whole.
|
||||||
@@ -272,3 +449,13 @@ func termWidth(prev int) int {
|
|||||||
}
|
}
|
||||||
return 80
|
return 80
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func termHeight(prev int) int {
|
||||||
|
if _, h, err := term.GetSize(int(os.Stdout.Fd())); err == nil && h > 0 {
|
||||||
|
return h
|
||||||
|
}
|
||||||
|
if prev > 0 {
|
||||||
|
return prev
|
||||||
|
}
|
||||||
|
return 24
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,9 +1,11 @@
|
|||||||
package progress
|
package progress
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
"unicode/utf8"
|
||||||
|
|
||||||
"github.com/hanbok/got/download"
|
"github.com/hanbok/got/download"
|
||||||
)
|
)
|
||||||
@@ -76,52 +78,148 @@ func TestRatesKeyedByID(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestLineOneSeeding: while seeding, render SEED(ratio), drop DL:, keep UL:.
|
func TestPctField(t *testing.T) {
|
||||||
func TestLineOneSeeding(t *testing.T) {
|
if got := pctField(download.Stat{Status: download.Active, Total: 100, Completed: 50}); got != " 50%" {
|
||||||
r := newReporter(true)
|
t.Errorf("active 50%% = %q, want ' 50%%'", got)
|
||||||
s := download.Stat{
|
|
||||||
ID: "h", Name: "f", IsBT: true, Status: download.Seeding,
|
|
||||||
Total: 1000, Completed: 1000, Uploaded: 1500, Conns: 3, Seeders: 2,
|
|
||||||
}
|
}
|
||||||
line := r.lineOne(s)
|
if got := pctField(download.Stat{Status: download.Seeding}); got != "100%" {
|
||||||
if !strings.Contains(line, "SEED(1.5)") {
|
t.Errorf("seeding = %q, want 100%%", got)
|
||||||
t.Errorf("missing SEED(1.5): %q", line)
|
|
||||||
}
|
}
|
||||||
if strings.Contains(line, "DL:") {
|
if got := pctField(download.Stat{Status: download.Active, Total: -1}); got != " --" {
|
||||||
t.Errorf("DL: should be dropped while seeding: %q", line)
|
t.Errorf("unknown total = %q, want ' --'", got)
|
||||||
}
|
|
||||||
if !strings.Contains(line, "UL:") {
|
|
||||||
t.Errorf("UL: should be kept while seeding: %q", line)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestLineOneCNSD: CN always shown; SD shown for torrents (even with 0 seeders),
|
// TestRowMetric: the rate column and trailing field per download state.
|
||||||
// absent for HTTP.
|
func TestRowMetric(t *testing.T) {
|
||||||
func TestLineOneCNSD(t *testing.T) {
|
|
||||||
r := newReporter(true)
|
r := newReporter(true)
|
||||||
bt := r.lineOne(download.Stat{ID: "h", Name: "f", IsBT: true, Total: 10, Completed: 1, Conns: 4, Seeders: 0})
|
if rate, tail := r.rowMetric(download.Stat{Status: download.Active, Total: 100, Completed: 50}, 10, 0); !strings.HasSuffix(rate, "/s") || tail == "" {
|
||||||
if !strings.Contains(bt, "CN:4") || !strings.Contains(bt, "SD:0") {
|
t.Errorf("active downloading: rate=%q tail=%q, want a …/s rate + an ETA", rate, tail)
|
||||||
t.Errorf("torrent line missing CN/SD: %q", bt)
|
|
||||||
}
|
}
|
||||||
http := r.lineOne(download.Stat{ID: "h2", Name: "f", IsBT: false, Total: 10, Completed: 1, Conns: 1})
|
if rate, tail := r.rowMetric(download.Stat{Status: download.Seeding, Completed: 1000, Uploaded: 1500}, 0, 0); rate != "seeding" || tail != "r1.50" {
|
||||||
if !strings.Contains(http, "CN:1") {
|
t.Errorf("seeding: rate=%q tail=%q, want seeding + r1.50 (share ratio)", rate, tail)
|
||||||
t.Errorf("http line missing CN: %q", http)
|
|
||||||
}
|
}
|
||||||
if strings.Contains(http, "SD:") {
|
if rate, _ := r.rowMetric(download.Stat{Status: download.Complete}, 0, 0); rate != "done" {
|
||||||
t.Errorf("http line should not show SD: %q", http)
|
t.Errorf("complete rate=%q, want done", rate)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestShortNameRune ensures CJK names are truncated on runes, not bytes.
|
// TestPeerFields: CN (connections/peers) and SD (seeders) show for active and
|
||||||
func TestShortNameRune(t *testing.T) {
|
// seeding downloads; SD is blank for HTTP; both blank when there are no live
|
||||||
|
// connections (queued).
|
||||||
|
func TestPeerFields(t *testing.T) {
|
||||||
|
if cn, sd := peerFields(download.Stat{Status: download.Active, IsBT: true, Conns: 18, Seeders: 4}); cn != "18" || sd != "4" {
|
||||||
|
t.Errorf("active torrent: cn=%q sd=%q, want 18 / 4", cn, sd)
|
||||||
|
}
|
||||||
|
if cn, sd := peerFields(download.Stat{Status: download.Active, IsBT: false, Conns: 5}); cn != "5" || sd != "" {
|
||||||
|
t.Errorf("active http: cn=%q sd=%q, want 5 / blank", cn, sd)
|
||||||
|
}
|
||||||
|
if cn, sd := peerFields(download.Stat{Status: download.Waiting, Conns: 0}); cn != "" || sd != "" {
|
||||||
|
t.Errorf("queued: cn=%q sd=%q, want blank / blank", cn, sd)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSizeField: completed/total when the total is known, just the completed
|
||||||
|
// bytes when it is not, and blank for a queued download.
|
||||||
|
func TestSizeField(t *testing.T) {
|
||||||
|
if got := sizeField(download.Stat{Status: download.Active, Completed: 512, Total: 2048}, false); got != "512B/2048B" {
|
||||||
|
t.Errorf("known total = %q, want 512B/2048B", got)
|
||||||
|
}
|
||||||
|
if got := sizeField(download.Stat{Status: download.Active, Completed: 512, Total: -1}, false); got != "512B" {
|
||||||
|
t.Errorf("unknown total = %q, want 512B", got)
|
||||||
|
}
|
||||||
|
if got := sizeField(download.Stat{Status: download.Waiting, Total: 2048}, false); got != "" {
|
||||||
|
t.Errorf("queued = %q, want blank", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestBlockTableAndFallback: several downloads render as summary+header+rows when
|
||||||
|
// they fit, and collapse to a single summary line when the window is too short.
|
||||||
|
// The fallback must leave the terminal's bottom row free (the stacked-summary
|
||||||
|
// redraw fix): summary+header+2 rows is 4 lines, so it needs height >= 5.
|
||||||
|
func TestBlockTableAndFallback(t *testing.T) {
|
||||||
|
r := newReporter(true)
|
||||||
|
stats := []download.Stat{
|
||||||
|
{ID: "a", Name: "a", Status: download.Active, Total: 100, Completed: 50},
|
||||||
|
{ID: "b", Name: "b", Status: download.Active, Total: 100, Completed: 10},
|
||||||
|
}
|
||||||
|
r.height = 24
|
||||||
|
if got := r.block(stats); len(got) != 4 { // summary + header + 2 rows
|
||||||
|
t.Errorf("table block = %d lines, want 4:\n%v", len(got), got)
|
||||||
|
}
|
||||||
|
// A single download renders the same way as several: summary + header + 1 row.
|
||||||
|
if got := r.block(stats[:1]); len(got) != 3 {
|
||||||
|
t.Errorf("single-download block = %d lines, want 3 (summary+header+row):\n%v", len(got), got)
|
||||||
|
}
|
||||||
|
r.height = 5 // 4 lines fit with the bottom row left free
|
||||||
|
if got := r.block(stats); len(got) != 4 {
|
||||||
|
t.Errorf("just-fits block = %d lines, want 4:\n%v", len(got), got)
|
||||||
|
}
|
||||||
|
r.height = 4 // one short: would fill to the bottom row, so fall back
|
||||||
|
if got := r.block(stats); len(got) != 1 {
|
||||||
|
t.Errorf("overflow block = %d lines, want 1 summary:\n%v", len(got), got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRenderRedraw: the TTY path draws the block, then on the next frame moves
|
||||||
|
// the cursor up over the previous frame and clears it before repainting.
|
||||||
|
func TestRenderRedraw(t *testing.T) {
|
||||||
|
snap := []download.Stat{
|
||||||
|
{ID: "a", Name: "alpha", IsBT: true, Status: download.Active, Total: 100, Completed: 50},
|
||||||
|
{ID: "b", Name: "beta", IsBT: true, Status: download.Seeding, Total: 100, Completed: 100, Uploaded: 200},
|
||||||
|
}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
r := &Reporter{
|
||||||
|
snap: func() []download.Stat { return snap },
|
||||||
|
human: true, w: &buf, tty: true, width: 200, height: 24,
|
||||||
|
win: map[string]*speedWindow{},
|
||||||
|
}
|
||||||
|
r.render(false)
|
||||||
|
first := buf.String()
|
||||||
|
for _, want := range []string{"downloading", "seeding", "alpha", "NAME"} {
|
||||||
|
if !strings.Contains(first, want) {
|
||||||
|
t.Fatalf("first frame missing %q:\n%q", want, first)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Contains(first, "|") {
|
||||||
|
t.Errorf("first frame should carry no progress bar:\n%q", first)
|
||||||
|
}
|
||||||
|
buf.Reset()
|
||||||
|
r.render(false)
|
||||||
|
second := buf.String()
|
||||||
|
if !strings.Contains(second, "\x1b[3A") { // up prevLines-1 = 4-1 (summary+header+2 rows)
|
||||||
|
t.Errorf("second frame should move cursor up 3 lines:\n%q", second)
|
||||||
|
}
|
||||||
|
if !strings.Contains(second, "\x1b[J") {
|
||||||
|
t.Errorf("second frame should clear to end of screen:\n%q", second)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestElideKeepsTail: names sharing a long prefix but differing at the end stay
|
||||||
|
// distinct — the failure the old leading truncation caused — and the result is
|
||||||
|
// capped at the column width on runes with the distinguishing tail preserved.
|
||||||
|
func TestElideKeepsTail(t *testing.T) {
|
||||||
|
a := elide("[Group] Long Common Series Name vol.1", nameW)
|
||||||
|
b := elide("[Group] Long Common Series Name vol.2", nameW)
|
||||||
|
if a == b {
|
||||||
|
t.Errorf("names differing only in the tail collapsed: %q == %q", a, b)
|
||||||
|
}
|
||||||
|
if n := utf8.RuneCountInString(a); n != nameW {
|
||||||
|
t.Errorf("elide rune count = %d, want %d (%q)", n, nameW, a)
|
||||||
|
}
|
||||||
|
if !strings.Contains(a, "…") {
|
||||||
|
t.Errorf("expected a middle ellipsis: %q", a)
|
||||||
|
}
|
||||||
|
if !strings.HasSuffix(a, "vol.1") {
|
||||||
|
t.Errorf("expected the distinguishing tail kept: %q", a)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestElideRune ensures CJK names are elided on runes, not bytes.
|
||||||
|
func TestElideRune(t *testing.T) {
|
||||||
name := strings.Repeat("あ", 30) // 30 runes, 90 bytes
|
name := strings.Repeat("あ", 30) // 30 runes, 90 bytes
|
||||||
got := shortName(name)
|
if n := utf8.RuneCountInString(elide(name, nameW)); n != nameW {
|
||||||
runes := []rune(got)
|
t.Errorf("elide rune count = %d, want %d", n, nameW)
|
||||||
if len(runes) != 20 {
|
|
||||||
t.Errorf("shortName rune count = %d, want 20", len(runes))
|
|
||||||
}
|
|
||||||
if runes[len(runes)-1] != '~' {
|
|
||||||
t.Errorf("expected trailing ~, got %q", got)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user