diff --git a/internal/protocol/http/fetcher.go b/internal/protocol/http/fetcher.go index 89804ce3..9eca0089 100644 --- a/internal/protocol/http/fetcher.go +++ b/internal/protocol/http/fetcher.go @@ -417,12 +417,26 @@ func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error { res.Range = true } - // Get content length from Content-Length header - contentLength := resp.Header.Get(base.HttpHeaderContentLength) - if contentLength != "" { - parse, err := strconv.ParseInt(contentLength, 10, 64) - if err == nil { - res.Size = parse + // A partial response describes one segment, not the whole resource. + reuseResolveBody := true + if resp.StatusCode == base.HttpCodePartialContent { + start, _, total, err := parseRangeResponse(resp) + if err != nil { + resp.Body.Close() + f.setState(stateError) + return err + } + res.Size = total + res.Range = true + // Prefetch is a contiguous prefix copied at offset zero. A segment + // starting elsewhere must be fetched again through the Range path. + reuseResolveBody = start == 0 + } else { + contentLength := resp.Header.Get(base.HttpHeaderContentLength) + if contentLength != "" { + if size, err := strconv.ParseInt(contentLength, 10, 64); err == nil { + res.Size = size + } } } @@ -467,7 +481,12 @@ func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error { // IMPORTANT: Keep the response body open for downloading in Start phase // This is crucial for one-time URLs that can only be accessed once f.resolveRespLock.Lock() - f.resolveResp = resp + if reuseResolveBody { + f.resolveResp = resp + } else { + resp.Body.Close() + f.prefetchDone.Store(true) + } f.resolveRespLock.Unlock() f.setState(stateResolved) @@ -479,7 +498,7 @@ func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error { // Start async prefetch in background (only for range-supported resources) // For non-range resources, the response will be used directly in Start - if res.Range { + if res.Range && reuseResolveBody { f.prefetchStopCh = make(chan struct{}) f.resolveFallback.Store(true) go f.asyncPrefetch() @@ -1544,27 +1563,41 @@ func (f *Fetcher) completeFromResolveFallback(conn *connection) bool { return true } -func validateRangeResponse(resp *http.Response, requestedStart int64, requestedEnd *int64, totalSize int64) (int64, int64, error) { +// parseRangeResponse validates a single byte range with a known complete length. +// Unknown totals are rejected: a segment length cannot stand in for resource size. +func parseRangeResponse(resp *http.Response) (int64, int64, int64, error) { contentRange := resp.Header.Get(base.HttpHeaderContentRange) fields := strings.Fields(contentRange) if len(fields) != 2 || !strings.EqualFold(fields[0], base.HttpHeaderBytes) { - return 0, 0, fmt.Errorf("%w: malformed Content-Range %q", errInvalidRangeResponse, contentRange) + return 0, 0, 0, fmt.Errorf("%w: malformed Content-Range %q", errInvalidRangeResponse, contentRange) } rangeAndTotal := strings.Split(fields[1], "/") if len(rangeAndTotal) != 2 { - return 0, 0, fmt.Errorf("%w: malformed Content-Range %q", errInvalidRangeResponse, contentRange) + return 0, 0, 0, fmt.Errorf("%w: malformed Content-Range %q", errInvalidRangeResponse, contentRange) } bounds := strings.Split(rangeAndTotal[0], "-") if len(bounds) != 2 { - return 0, 0, fmt.Errorf("%w: malformed Content-Range %q", errInvalidRangeResponse, contentRange) + return 0, 0, 0, fmt.Errorf("%w: malformed Content-Range %q", errInvalidRangeResponse, contentRange) } start, startErr := strconv.ParseInt(bounds[0], 10, 64) end, endErr := strconv.ParseInt(bounds[1], 10, 64) total, totalErr := strconv.ParseInt(rangeAndTotal[1], 10, 64) if startErr != nil || endErr != nil || totalErr != nil || start < 0 || end < start || total <= end { - return 0, 0, fmt.Errorf("%w: malformed Content-Range %q", errInvalidRangeResponse, contentRange) + return 0, 0, 0, fmt.Errorf("%w: malformed Content-Range %q", errInvalidRangeResponse, contentRange) + } + expectedLength := end - start + 1 + if resp.ContentLength >= 0 && resp.ContentLength != expectedLength { + return 0, 0, 0, fmt.Errorf("%w: Content-Length %d does not match range length %d", errInvalidRangeResponse, resp.ContentLength, expectedLength) + } + return start, end, total, nil +} + +func validateRangeResponse(resp *http.Response, requestedStart int64, requestedEnd *int64, totalSize int64) (int64, int64, error) { + start, end, total, err := parseRangeResponse(resp) + if err != nil { + return 0, 0, err } if start != requestedStart || (requestedEnd != nil && end != *requestedEnd) || (requestedEnd == nil && end != total-1) { return 0, 0, fmt.Errorf("%w: Content-Range %d-%d does not match requested range", errInvalidRangeResponse, start, end) @@ -1573,11 +1606,7 @@ func validateRangeResponse(resp *http.Response, requestedStart int64, requestedE return 0, 0, fmt.Errorf("%w: Content-Range total %d does not match resource size %d", errInvalidRangeResponse, total, totalSize) } - expectedLength := end - start + 1 - if resp.ContentLength >= 0 && resp.ContentLength != expectedLength { - return 0, 0, fmt.Errorf("%w: Content-Length %d does not match range length %d", errInvalidRangeResponse, resp.ContentLength, expectedLength) - } - return expectedLength, total, nil + return end - start + 1, total, nil } func (f *Fetcher) resolveOpenEndedRange(conn *connection, totalSize int64) error { diff --git a/internal/protocol/http/resolve_range_test.go b/internal/protocol/http/resolve_range_test.go new file mode 100644 index 00000000..d7e0b335 --- /dev/null +++ b/internal/protocol/http/resolve_range_test.go @@ -0,0 +1,119 @@ +package http + +import ( + "bytes" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "testing" + + "github.com/GopeedLab/gopeed/pkg/base" + fhttp "github.com/GopeedLab/gopeed/pkg/protocol/http" +) + +func TestResolveResponseSize(t *testing.T) { + for _, tc := range []struct { + name string + status int + length, contentRange string + size int64 + invalid bool + }{ + {"full", 200, "801571015", "", 801571015, false}, + {"partial_prefix", 206, "472321339", "bytes 0-472321338/801571015", 801571015, false}, + {"partial_nonzero", 206, "100000", "bytes 100000-199999/801571015", 801571015, false}, + {"second_report", 206, "335134409", "bytes 0-335134408/552140234", 552140234, false}, + {"length_mismatch", 206, "10", "bytes 0-19/100", 0, true}, + {"unknown_total", 206, "472321339", "bytes 0-472321338/*", 0, true}, + {"missing_range", 206, "10", "", 0, true}, + {"malformed_range", 206, "10", "bytes invalid", 0, true}, + {"invalid_bounds", 206, "10", "bytes 20-10/100", 0, true}, + {"invalid_total", 206, "10", "bytes 0-9/9", 0, true}, + {"overflow", 206, "10", "bytes 0-9/9223372036854775808", 0, true}, + {"chunked_partial", 206, "", "bytes 0-9/100", 100, false}, + {"unknown_full", 200, "", "", 0, false}, + } { + t.Run(tc.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if tc.length != "" { + w.Header().Set("Content-Length", tc.length) + } + if tc.contentRange != "" { + w.Header().Set("Content-Range", tc.contentRange) + } + w.WriteHeader(tc.status) + w.(http.Flusher).Flush() + })) + defer server.Close() + f := buildFetcher() + defer f.Close() + err := f.Resolve(&base.Request{URL: server.URL + "/file"}, &base.Options{Path: t.TempDir()}) + if tc.invalid { + if !errors.Is(err, errInvalidRangeResponse) { + t.Fatalf("Resolve error = %v, want invalid range", err) + } + if f.meta.Res != nil || f.resolveResp != nil || f.getState() != stateError { + t.Fatal("invalid response was retained or published") + } + return + } + if err != nil { + t.Fatal(err) + } + if f.meta.Res.Size != tc.size || f.meta.Res.Files[0].Size != tc.size { + t.Fatalf("resource/file size = %d/%d, want %d", f.meta.Res.Size, f.meta.Res.Files[0].Size, tc.size) + } + if tc.status == 206 && !f.meta.Res.Range { + t.Fatal("partial response must enable Range downloads") + } + }) + } +} + +func TestDownloadPartialResolve(t *testing.T) { + payload := make([]byte, 256*1024) + for i := range payload { + payload[i] = byte(i*31 + i/251) + } + for _, start := range []int{0, 100000} { + for _, connections := range []int{1, 4} { + t.Run(fmt.Sprintf("start_%d_connections_%d", start, connections), func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + begin, end := start, start+9999 + if h := r.Header.Get("Range"); h != "" { + if _, err := fmt.Sscanf(h, "bytes=%d-%d", &begin, &end); err != nil { + t.Errorf("invalid request range %q: %v", h, err) + w.WriteHeader(400) + return + } + } + w.Header().Set("Content-Length", fmt.Sprint(end-begin+1)) + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", begin, end, len(payload))) + w.WriteHeader(206) + w.Write(payload[begin : end+1]) + })) + defer server.Close() + f := buildFetcher() + defer f.Close() + if err := f.Resolve(&base.Request{URL: server.URL + "/file"}, &base.Options{Path: t.TempDir(), Name: "file", Extra: &fhttp.OptsExtra{Connections: connections}}); err != nil { + t.Fatal(err) + } + if err := f.Start(); err != nil { + t.Fatal(err) + } + if err := f.Wait(); err != nil { + t.Fatal(err) + } + got, err := os.ReadFile(f.meta.SingleFilepath()) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, payload) { + t.Fatalf("download content mismatch: got %d bytes, want %d", len(got), len(payload)) + } + }) + } + } +}