main: dedupe followed .torrent files by infohash
a .torrent fetched over http is added to the shared client in a second pass (followUps); two that name the same torrent would share one torrent on the refcount-less client, and the first to finish would Drop it out from under the other, stranding it at 0%. dedupe the followed files by infohash, the guard build() already applies to pass-1 sources.
This commit is contained in:
51
main.go
51
main.go
@@ -224,6 +224,24 @@ func dedupeTorrents(uris []string, follow bool) map[string]string {
|
||||
return dup
|
||||
}
|
||||
|
||||
// markInfoHash records the v1 infohash of the .torrent at path in seen and
|
||||
// reports whether it repeats one already seen, returning the path first recorded
|
||||
// under that infohash. Callers fail the duplicate rather than let two jobs share
|
||||
// one torrent on the client (see bt.SourceInfoHash for why that aliasing is
|
||||
// unsafe). A v2-only or unreadable infohash is never a duplicate — the dedup
|
||||
// can't see it, so it runs.
|
||||
func markInfoHash(seen map[metainfo.Hash]string, path string) (first string, dup bool) {
|
||||
h, ok := bt.SourceInfoHash(path, true)
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
if first, dup = seen[h]; dup {
|
||||
return first, true
|
||||
}
|
||||
seen[h] = path
|
||||
return "", false
|
||||
}
|
||||
|
||||
func build(opts *cli.Options, targets []target) ([]job, *torrent.Client, error) {
|
||||
flat := allURIs(targets)
|
||||
// Two sources naming the same torrent (a magnet and its own .torrent, say)
|
||||
@@ -358,6 +376,7 @@ func runJobs(ctx context.Context, eng *download.Engine, jobs []job) []download.R
|
||||
// so a regular download that merely ends in ".torrent" is left alone.
|
||||
func followUps(jobs []job, client *torrent.Client, bo bt.Options) []job {
|
||||
var out []job
|
||||
seen := map[metainfo.Hash]string{}
|
||||
for _, j := range jobs {
|
||||
h, ok := j.dl.(*httpdl.Download)
|
||||
if !ok || j.dl.Stat().Status != download.Complete {
|
||||
@@ -373,6 +392,17 @@ func followUps(jobs []job, client *torrent.Client, bo bt.Options) []job {
|
||||
if _, err := bt.Files(path); err != nil {
|
||||
continue // downloaded file is not a valid torrent
|
||||
}
|
||||
// Two fetched .torrent files that name the same torrent would share one
|
||||
// torrent on the refcount-less client: the first to finish Drops it out
|
||||
// from under the other, stranding it at 0%. build() dedupes pass-1 sources
|
||||
// but cannot reach a not-yet-fetched HTTP .torrent, so dedupe here too.
|
||||
if first, dup := markInfoHash(seen, path); dup {
|
||||
out = append(out, job{source: path, dl: failedDownload{
|
||||
name: path,
|
||||
err: fmt.Errorf("%w: same torrent as %s", errDuplicate, first),
|
||||
}})
|
||||
continue
|
||||
}
|
||||
out = append(out, job{source: path, dl: bt.New(client, path, true, bo)})
|
||||
}
|
||||
return out
|
||||
@@ -408,22 +438,23 @@ func allURIs(ts []target) []string {
|
||||
// gatherURIs collects download targets from the command line, --torrent-file,
|
||||
// and --input-file. Plain command-line http/https URIs are grouped into ONE
|
||||
// mirrored download by default (every command-line URI names the same file);
|
||||
// -Z/--force-sequential makes each its own download. Torrents and magnets
|
||||
// are always one target each; --input-file groups TAB-separated URIs per line.
|
||||
// -Z/--force-sequential makes each its own download. Torrents, magnets, and
|
||||
// .torrent URLs (fetched over HTTP then followed) are always one target each,
|
||||
// since distinct torrents are not mirrors of one another; --input-file groups
|
||||
// TAB-separated URIs per line.
|
||||
func gatherURIs(opts *cli.Options, cmdURIs []string) []target {
|
||||
var ts []target
|
||||
follow := opts.Bool("follow-torrent")
|
||||
seq := opts.Bool("force-sequential")
|
||||
var httpGroup []string
|
||||
for _, u := range cmdURIs {
|
||||
switch uriKind(u, follow) {
|
||||
case kindHTTP, kindFollow:
|
||||
if seq {
|
||||
ts = append(ts, target{uris: []string{u}})
|
||||
} else {
|
||||
httpGroup = append(httpGroup, u)
|
||||
}
|
||||
default:
|
||||
// Only plain HTTP URLs collapse into one mirror download (alternate sources
|
||||
// for the same file); -Z opts out. A .torrent URL (kindFollow), like a
|
||||
// torrent or magnet, is its own download — distinct torrents are never
|
||||
// mirrors of one another.
|
||||
if uriKind(u, follow) == kindHTTP && !seq {
|
||||
httpGroup = append(httpGroup, u)
|
||||
} else {
|
||||
ts = append(ts, target{uris: []string{u}})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user