httpdl: drop work-stealing segment pool for one worker per segment
The pool let an idle worker steal the back half of a slower in-flight segment, behind a mutex that also serialized every advance. At the default -x 1 it never fired, and it carried the trickiest invariant in the tree (minSplit >= readBuf keeps an owner's in-flight write out of the stolen tail). makeSegments already caps the segment count at the connection count, so the segments map one-to-one onto workers: give each its own goroutine, advance written with a plain atomic add, and snapshot the fixed slice with no lock. pool/newPool/acquire/steal/advance and the seg.owned field all go away. Net -156 lines. Behaviour at the user's flags is unchanged, except a slow mirror's tail segment is no longer rebalanced onto idle connections.
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
// Package httpdl downloads a single HTTP(S) resource over one or more
|
||||
// connections. The model is deliberately flat: split the file into byte-range
|
||||
// segments, hand them to a pool of worker goroutines, and have each worker
|
||||
// segments, give each segment its own worker goroutine, and have each worker
|
||||
// stream its range straight to disk with WriteAt (safe for concurrent,
|
||||
// non-overlapping writes — no shared seek, no mutex). Goroutines blocking on
|
||||
// real I/O keep the model simple: no segment manager, no piece storage, no
|
||||
@@ -715,36 +715,27 @@ func (d *Download) segmented(ctx context.Context, out string, total int64, etag,
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
p := newPool(segs, d.cfg.MinSplit)
|
||||
|
||||
// Periodically persist progress so a crash can resume.
|
||||
saveDone := make(chan struct{})
|
||||
go d.saveLoop(ctx, out, total, etag, lastmod, p, saveDone)
|
||||
go d.saveLoop(ctx, out, total, etag, lastmod, segs, saveDone)
|
||||
|
||||
// One worker per connection. A worker fetches segments until acquire runs
|
||||
// dry, which happens only once every remaining byte is owned; a worker that
|
||||
// finishes early steals the tail of a slower segment rather than sitting idle
|
||||
// while the last segment drains over a single connection.
|
||||
// One worker per segment. Segments are non-overlapping byte ranges written
|
||||
// with WriteAt, so the workers share no cursor and need no coordination — a
|
||||
// finished worker simply exits. The division (makeSegments / a resumed
|
||||
// control) fixes the parallelism up front; there is no rebalancing.
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
errOnce sync.Once
|
||||
runErr error
|
||||
)
|
||||
for w := 0; w < conns; w++ {
|
||||
for i := range segs {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
go func(s *seg) {
|
||||
defer wg.Done()
|
||||
for {
|
||||
s := p.acquire()
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
if err := d.fetchSeg(ctx, f, p, s, total); err != nil {
|
||||
errOnce.Do(func() { runErr = err; cancel() })
|
||||
return
|
||||
}
|
||||
if err := d.fetchSeg(ctx, f, s, total); err != nil {
|
||||
errOnce.Do(func() { runErr = err; cancel() })
|
||||
}
|
||||
}()
|
||||
}(&segs[i])
|
||||
}
|
||||
wg.Wait()
|
||||
cancel()
|
||||
@@ -753,7 +744,7 @@ func (d *Download) segmented(ctx context.Context, out string, total int64, etag,
|
||||
if runErr != nil {
|
||||
return runErr // keep the control file for a later -c
|
||||
}
|
||||
p.snapshot(d.primary(), total, etag, lastmod).save(out)
|
||||
snapshot(d.primary(), total, etag, lastmod, segs).save(out)
|
||||
removeControl(out)
|
||||
return nil
|
||||
}
|
||||
@@ -761,7 +752,7 @@ func (d *Download) segmented(ctx context.Context, out string, total int64, etag,
|
||||
// fetchSeg downloads one segment, retrying from its resume point on error.
|
||||
// Each attempt re-reads s.offset(), so a retry continues from the bytes already
|
||||
// written rather than restarting the range.
|
||||
func (d *Download) fetchSeg(ctx context.Context, f *os.File, p *pool, s *seg, total int64) error {
|
||||
func (d *Download) fetchSeg(ctx context.Context, f *os.File, s *seg, total int64) error {
|
||||
// Try each mirror in turn for this segment: a transient error retries the
|
||||
// same mirror under withRetries; an exhausted budget or a per-mirror permanent
|
||||
// error (a 404, a non-206, a length mismatch) falls over to the next mirror.
|
||||
@@ -773,12 +764,12 @@ func (d *Download) fetchSeg(ctx context.Context, f *os.File, p *pool, s *seg, to
|
||||
if s.done() {
|
||||
return nil
|
||||
}
|
||||
return d.fetchOnce(ctx, f, p, s, uri, total)
|
||||
return d.fetchOnce(ctx, f, s, uri, total)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func (d *Download) fetchOnce(ctx context.Context, f *os.File, p *pool, s *seg, uri string, total int64) error {
|
||||
func (d *Download) fetchOnce(ctx context.Context, f *os.File, s *seg, uri string, total int64) error {
|
||||
reqCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
req, err := d.request(reqCtx, uri, fmt.Sprintf("bytes=%d-%d", s.offset(), s.endOff()))
|
||||
@@ -815,7 +806,7 @@ func (d *Download) fetchOnce(ctx context.Context, f *os.File, p *pool, s *seg, u
|
||||
}
|
||||
body, stop := d.idleGuard(resp.Body, cancel)
|
||||
defer stop()
|
||||
return d.pump(ctx, f, p, s, body)
|
||||
return d.pump(ctx, f, s, body)
|
||||
}
|
||||
|
||||
// errIdleTimeout marks a transfer that stalled past the idle window, so callers
|
||||
@@ -986,10 +977,10 @@ func statusError(ctxMsg string, code int) error {
|
||||
}
|
||||
|
||||
// pump copies the response body into the file at the segment's running offset,
|
||||
// stopping at the segment end and respecting rate limits. The loop reads
|
||||
// remaining() afresh each turn, so a steal that shrinks s mid-transfer simply
|
||||
// ends the loop early at the new end, leaving the stolen tail to its new worker.
|
||||
func (d *Download) pump(ctx context.Context, f *os.File, p *pool, s *seg, body io.Reader) error {
|
||||
// stopping at the segment end and respecting rate limits. Only this worker writes
|
||||
// s.written, so the advance is a plain atomic add (Stat and the snapshot read it
|
||||
// atomically); the loop ends when the range is filled.
|
||||
func (d *Download) pump(ctx context.Context, f *os.File, s *seg, body io.Reader) error {
|
||||
buf := make([]byte, readBuf)
|
||||
for s.remaining() > 0 {
|
||||
n := int64(len(buf))
|
||||
@@ -1001,7 +992,7 @@ func (d *Download) pump(ctx context.Context, f *os.File, p *pool, s *seg, body i
|
||||
if _, werr := f.WriteAt(buf[:rd], s.offset()); werr != nil {
|
||||
return werr
|
||||
}
|
||||
p.advance(s, int64(rd))
|
||||
s.addWritten(int64(rd))
|
||||
atomic.AddInt64(&d.completed, int64(rd))
|
||||
d.throttle(ctx, rd)
|
||||
}
|
||||
@@ -1156,7 +1147,7 @@ func (d *Download) singleOnce(ctx context.Context, out, uri string, resumeFromDi
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Download) saveLoop(ctx context.Context, out string, total int64, etag, lastmod string, p *pool, done chan<- struct{}) {
|
||||
func (d *Download) saveLoop(ctx context.Context, out string, total int64, etag, lastmod string, segs []seg, done chan<- struct{}) {
|
||||
defer close(done)
|
||||
interval := d.cfg.AutoSaveInterval
|
||||
if interval <= 0 {
|
||||
@@ -1167,10 +1158,10 @@ func (d *Download) saveLoop(ctx context.Context, out string, total int64, etag,
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
p.snapshot(d.primary(), total, etag, lastmod).save(out)
|
||||
snapshot(d.primary(), total, etag, lastmod, segs).save(out)
|
||||
return
|
||||
case <-t.C:
|
||||
p.snapshot(d.primary(), total, etag, lastmod).save(out)
|
||||
snapshot(d.primary(), total, etag, lastmod, segs).save(out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user