From ee66235c36938f0b2510801617bfaf28708e96b3 Mon Sep 17 00:00:00 2001 From: Hojun-Cho Date: Sat, 20 Jun 2026 19:28:32 +0900 Subject: [PATCH] main: dedupe followed .torrent files by infohash a .torrent fetched over http is added to the shared client in a second pass (followUps); two that name the same torrent would share one torrent on the refcount-less client, and the first to finish would Drop it out from under the other, stranding it at 0%. dedupe the followed files by infohash, the guard build() already applies to pass-1 sources. --- main.go | 51 ++++++++++++++++++++++++++++------- main_test.go | 76 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 115 insertions(+), 12 deletions(-) diff --git a/main.go b/main.go index 535f238..5eee736 100644 --- a/main.go +++ b/main.go @@ -224,6 +224,24 @@ func dedupeTorrents(uris []string, follow bool) map[string]string { return dup } +// markInfoHash records the v1 infohash of the .torrent at path in seen and +// reports whether it repeats one already seen, returning the path first recorded +// under that infohash. Callers fail the duplicate rather than let two jobs share +// one torrent on the client (see bt.SourceInfoHash for why that aliasing is +// unsafe). A v2-only or unreadable infohash is never a duplicate — the dedup +// can't see it, so it runs. +func markInfoHash(seen map[metainfo.Hash]string, path string) (first string, dup bool) { + h, ok := bt.SourceInfoHash(path, true) + if !ok { + return "", false + } + if first, dup = seen[h]; dup { + return first, true + } + seen[h] = path + return "", false +} + func build(opts *cli.Options, targets []target) ([]job, *torrent.Client, error) { flat := allURIs(targets) // Two sources naming the same torrent (a magnet and its own .torrent, say) @@ -358,6 +376,7 @@ func runJobs(ctx context.Context, eng *download.Engine, jobs []job) []download.R // so a regular download that merely ends in ".torrent" is left alone. func followUps(jobs []job, client *torrent.Client, bo bt.Options) []job { var out []job + seen := map[metainfo.Hash]string{} for _, j := range jobs { h, ok := j.dl.(*httpdl.Download) if !ok || j.dl.Stat().Status != download.Complete { @@ -373,6 +392,17 @@ func followUps(jobs []job, client *torrent.Client, bo bt.Options) []job { if _, err := bt.Files(path); err != nil { continue // downloaded file is not a valid torrent } + // Two fetched .torrent files that name the same torrent would share one + // torrent on the refcount-less client: the first to finish Drops it out + // from under the other, stranding it at 0%. build() dedupes pass-1 sources + // but cannot reach a not-yet-fetched HTTP .torrent, so dedupe here too. + if first, dup := markInfoHash(seen, path); dup { + out = append(out, job{source: path, dl: failedDownload{ + name: path, + err: fmt.Errorf("%w: same torrent as %s", errDuplicate, first), + }}) + continue + } out = append(out, job{source: path, dl: bt.New(client, path, true, bo)}) } return out @@ -408,22 +438,23 @@ func allURIs(ts []target) []string { // gatherURIs collects download targets from the command line, --torrent-file, // and --input-file. Plain command-line http/https URIs are grouped into ONE // mirrored download by default (every command-line URI names the same file); -// -Z/--force-sequential makes each its own download. Torrents and magnets -// are always one target each; --input-file groups TAB-separated URIs per line. +// -Z/--force-sequential makes each its own download. Torrents, magnets, and +// .torrent URLs (fetched over HTTP then followed) are always one target each, +// since distinct torrents are not mirrors of one another; --input-file groups +// TAB-separated URIs per line. func gatherURIs(opts *cli.Options, cmdURIs []string) []target { var ts []target follow := opts.Bool("follow-torrent") seq := opts.Bool("force-sequential") var httpGroup []string for _, u := range cmdURIs { - switch uriKind(u, follow) { - case kindHTTP, kindFollow: - if seq { - ts = append(ts, target{uris: []string{u}}) - } else { - httpGroup = append(httpGroup, u) - } - default: + // Only plain HTTP URLs collapse into one mirror download (alternate sources + // for the same file); -Z opts out. A .torrent URL (kindFollow), like a + // torrent or magnet, is its own download — distinct torrents are never + // mirrors of one another. + if uriKind(u, follow) == kindHTTP && !seq { + httpGroup = append(httpGroup, u) + } else { ts = append(ts, target{uris: []string{u}}) } } diff --git a/main_test.go b/main_test.go index 4c4c678..2724550 100644 --- a/main_test.go +++ b/main_test.go @@ -11,6 +11,9 @@ import ( "syscall" "testing" + "github.com/anacrolix/torrent/bencode" + "github.com/anacrolix/torrent/metainfo" + "github.com/hanbok/got/bt" "github.com/hanbok/got/cli" "github.com/hanbok/got/download" "github.com/hanbok/got/httpdl" @@ -112,8 +115,8 @@ func TestDedupeTorrents(t *testing.T) { } // TestGatherURIsGrouping checks the command-line grouping: plain http URIs -// become one mirrored download by default, -Z splits them, and torrents/magnets -// stay separate. +// become one mirrored download by default, -Z splits them, and torrents, +// magnets, and .torrent URLs stay separate. func TestGatherURIsGrouping(t *testing.T) { parse := func(args ...string) []target { res, err := cli.Parse(append([]string{"--no-conf"}, args...)) @@ -131,6 +134,16 @@ func TestGatherURIsGrouping(t *testing.T) { if ts := parse("http://a/x", "magnet:?xt=urn:btih:abc"); len(ts) != 2 { t.Errorf("http + magnet: got %d targets, want 2 (magnet never mirrors)", len(ts)) } + // Distinct .torrent URLs are separate downloads, not mirrors of one file + // (follow-torrent defaults on, so each is fetched then followed). + if ts := parse("http://a/x.torrent", "http://b/y.torrent", "http://c/z.torrent"); len(ts) != 3 { + t.Errorf(".torrent URLs: got %d targets, want 3 separate downloads", len(ts)) + } + // A plain http URL still mirror-groups even alongside a .torrent URL: one + // http group of 1 + one torrent target = 2. + if ts := parse("http://a/x", "http://b/y.torrent"); len(ts) != 2 { + t.Errorf("http + .torrent: got %d targets, want 2", len(ts)) + } } // TestReadInputFileTAB checks the input-file grouping: TAB-separated URIs on @@ -152,6 +165,65 @@ func TestReadInputFileTAB(t *testing.T) { } } +// writeTorrent creates a .torrent file at dir/name whose info dict is built from +// a file of the given content and internal name. Two calls with the same +// internalName and content produce the same infohash regardless of the output +// filename, so the test can make two .torrent files that name one torrent. +func writeTorrent(t *testing.T, dir, name, internalName string, content []byte) string { + t.Helper() + data := filepath.Join(dir, internalName) + if err := os.WriteFile(data, content, 0o644); err != nil { + t.Fatal(err) + } + info := metainfo.Info{PieceLength: 32 * 1024} + if err := info.BuildFromFilePath(data); err != nil { + t.Fatal(err) + } + b, err := bencode.Marshal(info) + if err != nil { + t.Fatal(err) + } + tp := filepath.Join(dir, name) + f, err := os.Create(tp) + if err != nil { + t.Fatal(err) + } + defer f.Close() + if err := (&metainfo.MetaInfo{InfoBytes: b}).Write(f); err != nil { + t.Fatal(err) + } + return tp +} + +// TestFollowUpInfoHashDedup covers the fix for two fetched .torrent files that +// name the same torrent: the second must be flagged a duplicate so followUps +// fails it rather than letting both share — and Drop — one torrent on the client. +func TestFollowUpInfoHashDedup(t *testing.T) { + dir := t.TempDir() + a := writeTorrent(t, dir, "a.torrent", "same", []byte("AAAAAAAA")) + b := writeTorrent(t, dir, "b.torrent", "same", []byte("AAAAAAAA")) // same infohash as a + c := writeTorrent(t, dir, "c.torrent", "diff", []byte("BBBBBBBB")) // a distinct torrent + + // Sanity-check the fixture: a and b really share an infohash, c differs. + ha, _ := bt.SourceInfoHash(a, true) + hb, _ := bt.SourceInfoHash(b, true) + hc, _ := bt.SourceInfoHash(c, true) + if ha != hb || ha == hc { + t.Fatalf("fixture: want a==b!=c infohashes, got a=%v b=%v c=%v", ha, hb, hc) + } + + seen := map[metainfo.Hash]string{} + if first, dup := markInfoHash(seen, a); dup { + t.Fatalf("a is the first occurrence, wrongly flagged a duplicate of %s", first) + } + if first, dup := markInfoHash(seen, b); !dup || first != a { + t.Errorf("b should duplicate a: got first=%q dup=%v, want %q true", first, dup, a) + } + if _, dup := markInfoHash(seen, c); dup { + t.Errorf("c is a distinct torrent, wrongly flagged a duplicate") + } +} + func TestReportExit(t *testing.T) { canceled := download.Result{Name: "a", Err: context.Canceled} done := download.Result{Name: "b", Err: nil}