Hojun-Cho
ebc630b231
The pool let an idle worker steal the back half of a slower in-flight segment, behind a mutex that also serialized every advance. At the default -x 1 it never fired, and it carried the trickiest invariant in the tree (minSplit >= readBuf keeps an owner's in-flight write out of the stolen tail). makeSegments already caps the segment count at the connection count, so the segments map one-to-one onto workers: give each its own goroutine, advance written with a plain atomic add, and snapshot the fixed slice with no lock. pool/newPool/acquire/steal/advance and the seg.owned field all go away. Net -156 lines. Behaviour at the user's flags is unchanged, except a slow mirror's tail segment is no longer rebalanced onto idle connections.
111 lines
3.5 KiB
Go
111 lines
3.5 KiB
Go
package httpdl
|
|
|
|
import (
|
|
"encoding/json"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"sync/atomic"
|
|
)
|
|
|
|
// control is the on-disk resume state, a small JSON sidecar next to the output
|
|
// file (<out>.got). It is a Go-simple binary-free control file: enough to skip
|
|
// finished segments and resume partial ones, plus validators (Total +
|
|
// ETag/LastModified) so we never trust a stale file.
|
|
type control struct {
|
|
URL string `json:"url"`
|
|
Total int64 `json:"total"`
|
|
ETag string `json:"etag,omitempty"`
|
|
LastModified string `json:"last_modified,omitempty"`
|
|
Segs []segState `json:"segs"`
|
|
}
|
|
|
|
type segState struct {
|
|
Start int64 `json:"start"`
|
|
End int64 `json:"end"`
|
|
Written int64 `json:"written"`
|
|
}
|
|
|
|
func controlPath(out string) string { return out + ".got" }
|
|
|
|
// snapshot builds a control record from the live segments. The segment slice is
|
|
// fixed once the file is divided and each segment's frontier is owned by one
|
|
// worker, so reading written atomically gives a consistent record with no lock.
|
|
func snapshot(url string, total int64, etag, lastmod string, segs []seg) control {
|
|
c := control{URL: url, Total: total, ETag: etag, LastModified: lastmod, Segs: make([]segState, len(segs))}
|
|
for i := range segs {
|
|
c.Segs[i] = segState{segs[i].start, segs[i].endOff(), atomic.LoadInt64(&segs[i].written)}
|
|
}
|
|
return c
|
|
}
|
|
|
|
// save writes the control file atomically (temp + rename).
|
|
func (c control) save(out string) error {
|
|
data, err := json.Marshal(c)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := os.MkdirAll(filepath.Dir(out), 0o755); err != nil {
|
|
return err
|
|
}
|
|
tmp := controlPath(out) + ".tmp"
|
|
if err := os.WriteFile(tmp, data, 0o644); err != nil {
|
|
return err
|
|
}
|
|
return os.Rename(tmp, controlPath(out))
|
|
}
|
|
|
|
// loadControl reads a control file if it is present and still matches the
|
|
// download (same URL, length and validator). It returns nil when there is
|
|
// nothing trustworthy to resume from.
|
|
func loadControl(out, url string, total int64, etag, lastmod string) *control {
|
|
data, err := os.ReadFile(controlPath(out))
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
var c control
|
|
if json.Unmarshal(data, &c) != nil {
|
|
return nil
|
|
}
|
|
if c.URL != url || c.Total != total {
|
|
return nil
|
|
}
|
|
if (etag != "" || c.ETag != "") && c.ETag != etag {
|
|
return nil
|
|
}
|
|
if (lastmod != "" || c.LastModified != "") && c.LastModified != lastmod {
|
|
return nil
|
|
}
|
|
// Reject a control file whose segments do not exactly tile [0,total): a
|
|
// truncated/corrupted/hand-edited sidecar that still parses as JSON could
|
|
// otherwise mark a segment done() without its bytes on disk (inflated Written)
|
|
// or leave an un-downloaded hole, both of which would be reported as a complete
|
|
// file. We restart cleanly instead of trusting it.
|
|
if !validSegs(c.Segs, c.Total) {
|
|
return nil
|
|
}
|
|
return &c
|
|
}
|
|
|
|
// validSegs reports whether segs cover [0,total) with no gap or overlap and a
|
|
// sane written count for each. The recorded order is not sorted (a steal appends
|
|
// a tail), so we check coverage on a sorted copy.
|
|
func validSegs(segs []segState, total int64) bool {
|
|
if total <= 0 || len(segs) == 0 {
|
|
return false
|
|
}
|
|
sorted := append([]segState(nil), segs...)
|
|
sort.Slice(sorted, func(i, j int) bool { return sorted[i].Start < sorted[j].Start })
|
|
var next int64
|
|
for _, s := range sorted {
|
|
length := s.End - s.Start + 1
|
|
if s.Start != next || s.End < s.Start || s.Written < 0 || s.Written > length {
|
|
return false
|
|
}
|
|
next = s.End + 1
|
|
}
|
|
return next == total
|
|
}
|
|
|
|
func removeControl(out string) { os.Remove(controlPath(out)) }
|