first commit

httpdl/cookies.go

@@ -0,0 +1,177 @@
+package httpdl
+
+import (
+	"bufio"
+	"fmt"
+	"math"
+	"net/http"
+	"net/http/cookiejar"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// Cookie handling implements the --load-cookies / --save-cookies options, which
+// read and write the Netscape/Mozilla cookies.txt format (one tab-separated
+// cookie per line). We back it with the standard net/http/cookiejar so the
+// http.Client applies and tracks Set-Cookie headers automatically; the file
+// format is just the on-disk representation of that jar.
+
+// loadCookieJar builds a cookie jar seeded from a Netscape cookies.txt file. A
+// missing file yields an empty jar (not an error): there are simply no cookies
+// to load. A malformed line is skipped, giving per-line tolerance.
+func loadCookieJar(path string) (*cookiejar.Jar, error) {
+	jar, err := cookiejar.New(nil)
+	if err != nil {
+		return nil, err
+	}
+	if path == "" {
+		return jar, nil
+	}
+	f, err := os.Open(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return jar, nil // nothing to load yet
+		}
+		return nil, fmt.Errorf("load-cookies %s: %w", path, err)
+	}
+	defer f.Close()
+
+	// Group cookies by the URL their domain implies, since SetCookies keys on a
+	// URL. We synthesise a representative URL per (domain, secure, path).
+	byURL := map[string][]*http.Cookie{}
+	urls := map[string]*url.URL{}
+	sc := bufio.NewScanner(f)
+	for sc.Scan() {
+		line := sc.Text()
+		c, u := parseNetscapeCookie(line)
+		if c == nil {
+			continue
+		}
+		key := u.String()
+		if _, ok := urls[key]; !ok {
+			urls[key] = u
+		}
+		byURL[key] = append(byURL[key], c)
+	}
+	if err := sc.Err(); err != nil {
+		return nil, fmt.Errorf("load-cookies %s: %w", path, err)
+	}
+	for key, cs := range byURL {
+		jar.SetCookies(urls[key], cs)
+	}
+	return jar, nil
+}
+
+// parseNetscapeCookie parses one cookies.txt line into a cookie plus the URL it
+// belongs to, or (nil, nil) for a comment, blank or malformed line. The format
+// is seven tab-separated fields:
+//
+//	domain  includeSubdomains  path  secure  expiry  name  value
+//
+// Curl marks HttpOnly cookies with a "#HttpOnly_" line prefix rather than a
+// real comment, so we recognise that prefix instead of dropping the line.
+// Honouring HttpOnly_ rather than skipping every "#" line never loses a cookie.
+func parseNetscapeCookie(line string) (*http.Cookie, *url.URL) {
+	httpOnly := false
+	if strings.HasPrefix(line, "#HttpOnly_") {
+		httpOnly = true
+		line = strings.TrimPrefix(line, "#HttpOnly_")
+	} else if strings.HasPrefix(line, "#") {
+		return nil, nil
+	}
+	if strings.TrimSpace(line) == "" {
+		return nil, nil
+	}
+	f := strings.Split(line, "\t")
+	if len(f) < 6 {
+		return nil, nil
+	}
+	domain := f[0]
+	hostOnly := !strings.EqualFold(f[1], "TRUE")
+	path := f[2]
+	secure := strings.EqualFold(f[3], "TRUE")
+	name := f[5]
+	value := ""
+	if len(f) >= 7 {
+		value = f[6]
+	}
+	if name == "" || domain == "" {
+		return nil, nil
+	}
+
+	c := &http.Cookie{
+		Name:     name,
+		Value:    value,
+		Path:     path,
+		Secure:   secure,
+		HttpOnly: httpOnly,
+	}
+	// A non-host-only cookie (includeSubdomains TRUE) carries a Domain attribute
+	// so the jar applies it to subdomains; a host-only one leaves Domain empty so
+	// the jar binds it to the exact request host.
+	if !hostOnly {
+		c.Domain = domain
+	}
+	// expiry 0 means a session cookie; otherwise it is a Unix timestamp. The jar
+	// drops already-expired cookies on load.
+	if exp, err := strconv.ParseInt(strings.TrimSpace(f[4]), 10, 64); err == nil && exp > 0 {
+		c.Expires = time.Unix(exp, 0)
+	}
+
+	// The jar needs a URL whose host and scheme match the cookie. A leading dot
+	// denotes a domain cookie; strip it for the host but keep the cookie Domain
+	// so the jar treats it as domain-wide.
+	host := strings.TrimPrefix(domain, ".")
+	scheme := "http"
+	if secure {
+		scheme = "https"
+	}
+	u := &url.URL{Scheme: scheme, Host: host, Path: path}
+	return c, u
+}
+
+// saveCookieJar writes every cookie the jar holds (for the hosts we have
+// contacted) back to path in Netscape format, implementing --save-cookies on
+// exit. Because net/http/cookiejar exposes cookies only per URL, we ask it for
+// the cookies of each host we recorded a request against.
+func saveCookieJar(jar *cookiejar.Jar, urls []*url.URL, path string) error {
+	if path == "" || jar == nil {
+		return nil
+	}
+	var b strings.Builder
+	b.WriteString("# Netscape HTTP Cookie File\n")
+	seen := map[string]bool{}
+	for _, u := range urls {
+		for _, c := range jar.Cookies(u) {
+			// Cookies() drops the domain/path/expiry, so reconstruct sensible
+			// values: host-only domain, root path, far-future expiry. This is the
+			// best a host-only readback can do; it round-trips the name/value pair
+			// that authenticated downloads actually need.
+			key := u.Host + "\x00" + c.Name
+			if seen[key] {
+				continue
+			}
+			seen[key] = true
+			expiry := int64(math.MaxInt32) // session-stable far future
+			line := strings.Join([]string{
+				u.Host, "FALSE", "/",
+				boolToTRUE(u.Scheme == "https"),
+				strconv.FormatInt(expiry, 10),
+				c.Name, c.Value,
+			}, "\t")
+			b.WriteString(line)
+			b.WriteByte('\n')
+		}
+	}
+	return os.WriteFile(path, []byte(b.String()), 0o600)
+}
+
+func boolToTRUE(b bool) string {
+	if b {
+		return "TRUE"
+	}
+	return "FALSE"
+}