package httpdl import ( "bufio" "fmt" "math" "net/http" "net/http/cookiejar" "net/url" "os" "strconv" "strings" "time" ) // Cookie handling implements the --load-cookies / --save-cookies options, which // read and write the Netscape/Mozilla cookies.txt format (one tab-separated // cookie per line). We back it with the standard net/http/cookiejar so the // http.Client applies and tracks Set-Cookie headers automatically; the file // format is just the on-disk representation of that jar. // loadCookieJar builds a cookie jar seeded from a Netscape cookies.txt file. A // missing file yields an empty jar (not an error): there are simply no cookies // to load. A malformed line is skipped, giving per-line tolerance. func loadCookieJar(path string) (*cookiejar.Jar, error) { jar, err := cookiejar.New(nil) if err != nil { return nil, err } if path == "" { return jar, nil } f, err := os.Open(path) if err != nil { if os.IsNotExist(err) { return jar, nil // nothing to load yet } return nil, fmt.Errorf("load-cookies %s: %w", path, err) } defer f.Close() // Group cookies by the URL their domain implies, since SetCookies keys on a // URL. We synthesise a representative URL per (domain, secure, path). byURL := map[string][]*http.Cookie{} urls := map[string]*url.URL{} sc := bufio.NewScanner(f) for sc.Scan() { line := sc.Text() c, u := parseNetscapeCookie(line) if c == nil { continue } key := u.String() if _, ok := urls[key]; !ok { urls[key] = u } byURL[key] = append(byURL[key], c) } if err := sc.Err(); err != nil { return nil, fmt.Errorf("load-cookies %s: %w", path, err) } for key, cs := range byURL { jar.SetCookies(urls[key], cs) } return jar, nil } // parseNetscapeCookie parses one cookies.txt line into a cookie plus the URL it // belongs to, or (nil, nil) for a comment, blank or malformed line. The format // is seven tab-separated fields: // // domain includeSubdomains path secure expiry name value // // Curl marks HttpOnly cookies with a "#HttpOnly_" line prefix rather than a // real comment, so we recognise that prefix instead of dropping the line. // Honouring HttpOnly_ rather than skipping every "#" line never loses a cookie. func parseNetscapeCookie(line string) (*http.Cookie, *url.URL) { httpOnly := false if strings.HasPrefix(line, "#HttpOnly_") { httpOnly = true line = strings.TrimPrefix(line, "#HttpOnly_") } else if strings.HasPrefix(line, "#") { return nil, nil } if strings.TrimSpace(line) == "" { return nil, nil } f := strings.Split(line, "\t") if len(f) < 6 { return nil, nil } domain := f[0] hostOnly := !strings.EqualFold(f[1], "TRUE") path := f[2] secure := strings.EqualFold(f[3], "TRUE") name := f[5] value := "" if len(f) >= 7 { value = f[6] } if name == "" || domain == "" { return nil, nil } c := &http.Cookie{ Name: name, Value: value, Path: path, Secure: secure, HttpOnly: httpOnly, } // A non-host-only cookie (includeSubdomains TRUE) carries a Domain attribute // so the jar applies it to subdomains; a host-only one leaves Domain empty so // the jar binds it to the exact request host. if !hostOnly { c.Domain = domain } // expiry 0 means a session cookie; otherwise it is a Unix timestamp. The jar // drops already-expired cookies on load. if exp, err := strconv.ParseInt(strings.TrimSpace(f[4]), 10, 64); err == nil && exp > 0 { c.Expires = time.Unix(exp, 0) } // The jar needs a URL whose host and scheme match the cookie. A leading dot // denotes a domain cookie; strip it for the host but keep the cookie Domain // so the jar treats it as domain-wide. host := strings.TrimPrefix(domain, ".") scheme := "http" if secure { scheme = "https" } u := &url.URL{Scheme: scheme, Host: host, Path: path} return c, u } // saveCookieJar writes every cookie the jar holds (for the hosts we have // contacted) back to path in Netscape format, implementing --save-cookies on // exit. Because net/http/cookiejar exposes cookies only per URL, we ask it for // the cookies of each host we recorded a request against. func saveCookieJar(jar *cookiejar.Jar, urls []*url.URL, path string) error { if path == "" || jar == nil { return nil } var b strings.Builder b.WriteString("# Netscape HTTP Cookie File\n") seen := map[string]bool{} for _, u := range urls { for _, c := range jar.Cookies(u) { // Cookies() drops the domain/path/expiry, so reconstruct sensible // values: host-only domain, root path, far-future expiry. This is the // best a host-only readback can do; it round-trips the name/value pair // that authenticated downloads actually need. key := u.Host + "\x00" + c.Name if seen[key] { continue } seen[key] = true expiry := int64(math.MaxInt32) // session-stable far future line := strings.Join([]string{ u.Host, "FALSE", "/", boolToTRUE(u.Scheme == "https"), strconv.FormatInt(expiry, 10), c.Name, c.Value, }, "\t") b.WriteString(line) b.WriteByte('\n') } } return os.WriteFile(path, []byte(b.String()), 0o600) } func boolToTRUE(b bool) string { if b { return "TRUE" } return "FALSE" }