got/httpdl/cookies.go

package httpdl

import (
	"bufio"
	"fmt"
	"math"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"
)

// Cookie handling implements the --load-cookies / --save-cookies options, which
// read and write the Netscape/Mozilla cookies.txt format (one tab-separated
// cookie per line). We back it with the standard net/http/cookiejar so the
// http.Client applies and tracks Set-Cookie headers automatically; the file
// format is just the on-disk representation of that jar.

// loadCookieJar builds a cookie jar seeded from a Netscape cookies.txt file. A
// missing file yields an empty jar (not an error): there are simply no cookies
// to load. A malformed line is skipped, giving per-line tolerance.
func loadCookieJar(path string) (*cookiejar.Jar, error) {
	jar, err := cookiejar.New(nil)
	if err != nil {
		return nil, err
	}
	if path == "" {
		return jar, nil
	}
	f, err := os.Open(path)
	if err != nil {
		if os.IsNotExist(err) {
			return jar, nil // nothing to load yet
		}
		return nil, fmt.Errorf("load-cookies %s: %w", path, err)
	}
	defer f.Close()

	// Group cookies by the URL their domain implies, since SetCookies keys on a
	// URL. We synthesise a representative URL per (domain, secure, path).
	byURL := map[string][]*http.Cookie{}
	urls := map[string]*url.URL{}
	sc := bufio.NewScanner(f)
	for sc.Scan() {
		line := sc.Text()
		c, u := parseNetscapeCookie(line)
		if c == nil {
			continue
		}
		key := u.String()
		if _, ok := urls[key]; !ok {
			urls[key] = u
		}
		byURL[key] = append(byURL[key], c)
	}
	if err := sc.Err(); err != nil {
		return nil, fmt.Errorf("load-cookies %s: %w", path, err)
	}
	for key, cs := range byURL {
		jar.SetCookies(urls[key], cs)
	}
	return jar, nil
}

// parseNetscapeCookie parses one cookies.txt line into a cookie plus the URL it
// belongs to, or (nil, nil) for a comment, blank or malformed line. The format
// is seven tab-separated fields:
//
//	domain  includeSubdomains  path  secure  expiry  name  value
//
// Curl marks HttpOnly cookies with a "#HttpOnly_" line prefix rather than a
// real comment, so we recognise that prefix instead of dropping the line.
// Honouring HttpOnly_ rather than skipping every "#" line never loses a cookie.
func parseNetscapeCookie(line string) (*http.Cookie, *url.URL) {
	httpOnly := false
	if strings.HasPrefix(line, "#HttpOnly_") {
		httpOnly = true
		line = strings.TrimPrefix(line, "#HttpOnly_")
	} else if strings.HasPrefix(line, "#") {
		return nil, nil
	}
	if strings.TrimSpace(line) == "" {
		return nil, nil
	}
	f := strings.Split(line, "\t")
	if len(f) < 6 {
		return nil, nil
	}
	domain := f[0]
	hostOnly := !strings.EqualFold(f[1], "TRUE")
	path := f[2]
	secure := strings.EqualFold(f[3], "TRUE")
	name := f[5]
	value := ""
	if len(f) >= 7 {
		value = f[6]
	}
	if name == "" || domain == "" {
		return nil, nil
	}

	c := &http.Cookie{
		Name:     name,
		Value:    value,
		Path:     path,
		Secure:   secure,
		HttpOnly: httpOnly,
	}
	// A non-host-only cookie (includeSubdomains TRUE) carries a Domain attribute
	// so the jar applies it to subdomains; a host-only one leaves Domain empty so
	// the jar binds it to the exact request host.
	if !hostOnly {
		c.Domain = domain
	}
	// expiry 0 means a session cookie; otherwise it is a Unix timestamp. The jar
	// drops already-expired cookies on load.
	if exp, err := strconv.ParseInt(strings.TrimSpace(f[4]), 10, 64); err == nil && exp > 0 {
		c.Expires = time.Unix(exp, 0)
	}

	// The jar needs a URL whose host and scheme match the cookie. A leading dot
	// denotes a domain cookie; strip it for the host but keep the cookie Domain
	// so the jar treats it as domain-wide.
	host := strings.TrimPrefix(domain, ".")
	scheme := "http"
	if secure {
		scheme = "https"
	}
	u := &url.URL{Scheme: scheme, Host: host, Path: path}
	return c, u
}

// saveCookieJar writes every cookie the jar holds (for the hosts we have
// contacted) back to path in Netscape format, implementing --save-cookies on
// exit. Because net/http/cookiejar exposes cookies only per URL, we ask it for
// the cookies of each host we recorded a request against.
func saveCookieJar(jar *cookiejar.Jar, urls []*url.URL, path string) error {
	if path == "" || jar == nil {
		return nil
	}
	var b strings.Builder
	b.WriteString("# Netscape HTTP Cookie File\n")
	seen := map[string]bool{}
	for _, u := range urls {
		for _, c := range jar.Cookies(u) {
			// Cookies() drops the domain/path/expiry, so reconstruct sensible
			// values: host-only domain, root path, far-future expiry. This is the
			// best a host-only readback can do; it round-trips the name/value pair
			// that authenticated downloads actually need.
			key := u.Host + "\x00" + c.Name
			if seen[key] {
				continue
			}
			seen[key] = true
			expiry := int64(math.MaxInt32) // session-stable far future
			line := strings.Join([]string{
				u.Host, "FALSE", "/",
				boolToTRUE(u.Scheme == "https"),
				strconv.FormatInt(expiry, 10),
				c.Name, c.Value,
			}, "\t")
			b.WriteString(line)
			b.WriteByte('\n')
		}
	}
	return os.WriteFile(path, []byte(b.String()), 0o600)
}

func boolToTRUE(b bool) string {
	if b {
		return "TRUE"
	}
	return "FALSE"
}