feat(http): support advisory extension download sizes (#1599)

This commit is contained in:
Levi authored and GitHub committed 2026-10-22 23:35:01 +08:00
1 parent 4214765d39
commit d125d4ab75
17 files changed
+1047 -182

No files matched your search

+91 -115
View File
@@ -316,13 +316,17 @@ type Fetcher struct {
resolveFallback atomic.Bool // Resolve prefetch is still available until the first Range request takes over
// Target file
file *os.File
fileMu sync.Mutex
redirectURL string
redirectLock sync.Mutex
ifRange string
rangeReprobeEligible bool
rangeValidatorPinned bool
file *os.File
fileMu sync.Mutex
redirectURL string
redirectLock sync.Mutex
ifRange string
rangeReprobeEligible bool
rangeValidatorPinned bool
// sizeKnown distinguishes a display estimate from an HTTP-confirmed length,
// including zero. Nil preserves the interpretation of legacy snapshots.
sizeKnown *bool
// An interrupted chunked sequential response must restart from byte zero.
sequentialSizeUnknown bool
// Lifecycle control
@@ -369,6 +373,40 @@ func (f *Fetcher) setState(s fetcherState) {
f.state.Store(int32(s))
}
// Size state is read under connMu or before download workers are started.
func (f *Fetcher) hasKnownSize() bool {
if f.sizeKnown != nil {
return *f.sizeKnown
}
return f.meta != nil && f.meta.Res != nil && f.meta.Res.Size > 0 && !f.sequentialSizeUnknown
}
func (f *Fetcher) knownSize() int64 {
if f.hasKnownSize() {
return f.meta.Res.Size
}
return 0
}
func (f *Fetcher) sizeHint() int64 {
if f.meta.Opts != nil && f.meta.Opts.Size > 0 {
return f.meta.Opts.Size
}
return 0
}
// setResourceSizeLocked publishes the value and its authority in one snapshot.
func (f *Fetcher) setResourceSizeLocked(size int64, known bool) {
if !known {
size = f.sizeHint()
}
f.sizeKnown = &known
f.meta.Res.Size = size
if len(f.meta.Res.Files) > 0 && f.meta.Res.Files[0] != nil {
f.meta.Res.Files[0].Size = size
}
}
// updateMaxConnTime updates maxConnTime if the new duration is larger
func (f *Fetcher) updateMaxConnTime(d time.Duration) {
newVal := int64(d)
@@ -447,6 +485,13 @@ func (f *Fetcher) Resolve(req *base.Request) error {
}
}
}
// A hint is only a display value, even when the origin advertises Range.
known := resp.StatusCode == base.HttpCodePartialContent || resp.ContentLength >= 0
f.sizeKnown = &known
f.sequentialSizeUnknown = !res.Range && !known
if !known {
res.Size = f.sizeHint()
}
// Parse last modified time
var lastModifiedTime *time.Time
@@ -803,7 +848,7 @@ func (f *Fetcher) doStart() error {
_, err = os.Stat(name)
if err != nil {
if os.IsNotExist(err) {
file, err = f.ctl.Touch(name, f.meta.Res.Size)
file, err = f.ctl.Touch(name, f.knownSize())
} else {
return err
}
@@ -821,7 +866,7 @@ func (f *Fetcher) doStart() error {
// For non-range resources, the response will be used directly
var prefetchedBytes int64
if f.meta.Res.Range {
resolveComplete := f.meta.Res.Size > 0 && f.prefetchDone.Load() && f.prefetchErr == nil && f.prefetchSize.Load() >= f.meta.Res.Size
resolveComplete := f.knownSize() > 0 && f.prefetchDone.Load() && f.prefetchErr == nil && f.prefetchSize.Load() >= f.knownSize()
if f.resolveFallback.Load() && !resolveComplete {
// Keep Resolve downloading until the first Range response has been
// validated. The Range request starts at a stable snapshot; a valid
@@ -993,7 +1038,7 @@ func (f *Fetcher) expandConnections() {
return
}
totalSize := f.meta.Res.Size
totalSize := f.knownSize()
f.connMu.Lock()
@@ -1284,7 +1329,7 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
rangeMode := f.meta.Res.Range
ifRange := ""
if resumeProbe {
end := f.meta.Res.Size - 1
end := f.knownSize() - 1
rangeEnd = &end
ifRange = f.ifRange
} else if rangeMode && (f.rangeValidatorPinned || f.resolveFallback.Load()) {
@@ -1364,7 +1409,7 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
}
if rangeRequested && resp.StatusCode == base.HttpCodePartialContent {
var responseTotal int64
expectedResponseLength, responseTotal, err = validateRangeResponse(resp, rangeStart, rangeEnd, f.meta.Res.Size)
expectedResponseLength, responseTotal, err = validateRangeResponse(resp, rangeStart, rangeEnd, f.knownSize())
if err != nil {
resp.Body.Close()
return err
@@ -1389,7 +1434,7 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
f.rangeValidatorPinned = true
f.sequentialSizeUnknown = false
conn.Chunk.Begin = 0
conn.Chunk.setEnd(f.meta.Res.Size - 1)
conn.Chunk.setEnd(f.knownSize() - 1)
f.connMu.Unlock()
}
}
@@ -1414,7 +1459,7 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
resp.Body.Close()
return err
}
} else if f.meta.Res.Size > 0 {
} else if f.hasKnownSize() && f.meta.Res.Size > 0 {
expectedResponseLength = f.meta.Res.Size
if resp.ContentLength >= 0 && resp.ContentLength != expectedResponseLength {
resp.Body.Close()
@@ -1550,7 +1595,7 @@ func (f *Fetcher) commitResolveTakeover(conn *connection, requestedStart int64)
}
prefetched := f.stopPrefetchAndCopyData()
if prefetched < requestedStart || prefetched > f.meta.Res.Size {
if prefetched < requestedStart || prefetched > f.knownSize() {
return 0, fmt.Errorf("resolve prefetch advanced to invalid offset %d for range start %d", prefetched, requestedStart)
}
f.resolveDataPos.Store(prefetched)
@@ -1581,7 +1626,7 @@ func (f *Fetcher) completeFromResolveFallback(conn *connection) bool {
}
time.Sleep(10 * time.Millisecond)
}
knownSize := f.meta.Res.Size > 0
knownSize := f.hasKnownSize()
if f.prefetchErr != nil || (knownSize && f.prefetchSize.Load() != f.meta.Res.Size) {
f.cleanupPrefetchFile()
return false
@@ -1595,10 +1640,7 @@ func (f *Fetcher) completeFromResolveFallback(conn *connection) bool {
f.connMu.Lock()
if !knownSize {
f.meta.Res.Size = prefetched
if len(f.meta.Res.Files) > 0 && f.meta.Res.Files[0] != nil {
f.meta.Res.Files[0].Size = prefetched
}
f.setResourceSizeLocked(prefetched, true)
}
f.meta.Res.Range = false
f.rangeReprobeEligible = false
@@ -1680,10 +1722,7 @@ func (f *Fetcher) resolveOpenEndedRange(conn *connection, totalSize int64) error
f.connMu.Unlock()
return err
}
f.meta.Res.Size = totalSize
if len(f.meta.Res.Files) > 0 && f.meta.Res.Files[0] != nil {
f.meta.Res.Files[0].Size = totalSize
}
f.setResourceSizeLocked(totalSize, true)
conn.Chunk.setEnd(totalSize - 1)
f.connMu.Unlock()
return nil
@@ -1755,10 +1794,7 @@ func (f *Fetcher) resizeSequentialResource(size int64) error {
return err
}
f.meta.Res.Size = size
if len(f.meta.Res.Files) > 0 {
f.meta.Res.Files[0].Size = size
}
f.setResourceSizeLocked(size, true)
f.sequentialSizeUnknown = false
f.connMu.Unlock()
return nil
@@ -1783,10 +1819,7 @@ func (f *Fetcher) beginUnknownSizeSequentialResource() error {
return err
}
f.meta.Res.Size = 0
if len(f.meta.Res.Files) > 0 {
f.meta.Res.Files[0].Size = 0
}
f.setResourceSizeLocked(0, false)
f.sequentialSizeUnknown = true
f.ifRange = ""
f.rangeValidatorPinned = false
@@ -1870,11 +1903,9 @@ func (f *Fetcher) runConnectionWithResolveResp(conn *connection) {
if err != nil {
if err == io.EOF {
f.connMu.Lock()
if f.meta.Res.Size == 0 {
f.meta.Res.Size = conn.Downloaded
if len(f.meta.Res.Files) > 0 && f.meta.Res.Files[0] != nil {
f.meta.Res.Files[0].Size = conn.Downloaded
}
if !f.hasKnownSize() {
f.setResourceSizeLocked(conn.Downloaded, true)
f.sequentialSizeUnknown = false
}
conn.Completed = true
conn.State = connCompleted
@@ -1913,78 +1944,15 @@ func (f *Fetcher) runConnectionFallback(conn *connection) {
}
f.connMu.Lock()
if retries > 0 {
f.resetConnectionForRestart(conn)
}
conn.State = connConnecting
f.connMu.Unlock()
err := func() error {
httpReq, err := f.buildRequest(conn.ctx, f.meta.Req)
if err != nil {
return err
}
// Record connection start time for adaptive timeout tracking
connStartTime := time.Now()
resp, err := client.Do(httpReq)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != base.HttpCodeOK && resp.StatusCode != base.HttpCodePartialContent {
return NewRequestError(resp.StatusCode)
}
// Record successful connection time for adaptive timeout
f.updateMaxConnTime(time.Since(connStartTime))
f.connMu.Lock()
conn.State = connDownloading
conn.failed = false
f.connMu.Unlock()
f.primaryReadyOnce.Do(func() {
close(f.primaryReadyCh)
})
if f.slowStart != nil {
f.slowStart.onConnectSuccess()
}
reader := f.responseReader(resp)
for {
if conn.ctx.Err() != nil {
return conn.ctx.Err()
}
n, err := reader.Read(buf)
if n > 0 {
f.connMu.Lock()
writeOffset := conn.Chunk.Downloaded
f.connMu.Unlock()
f.fileMu.Lock()
if f.file != nil {
_, writeErr := f.file.WriteAt(buf[:n], writeOffset)
if writeErr != nil {
f.fileMu.Unlock()
return writeErr
}
}
f.fileMu.Unlock()
f.connMu.Lock()
conn.Chunk.Downloaded += int64(n)
conn.Downloaded += int64(n)
f.connMu.Unlock()
}
if err != nil {
if err == io.EOF {
return nil
}
return err
}
}
}()
// Restored tasks without a reusable Resolve response need the same
// length handling and restart semantics as other sequential requests.
err := f.downloadChunkOnce(conn, client, buf)
if err == nil {
f.connMu.Lock()
@@ -2133,14 +2101,14 @@ func (f *Fetcher) resetConnectionForRestart(conn *connection) {
// hasSequentialPrefixLocked reports whether a single sequential connection has
// a useful contiguous prefix. The caller must hold connMu.
func (f *Fetcher) hasSequentialPrefixLocked(conn *connection) bool {
if f.meta == nil || f.meta.Res == nil || f.meta.Res.Range || !f.rangeReprobeEligible || f.meta.Res.Size <= 0 {
if f.meta == nil || f.meta.Res == nil || f.meta.Res.Range || !f.rangeReprobeEligible || f.knownSize() <= 0 {
return false
}
if conn == nil || conn.ID != 0 || len(f.connections) != 1 || conn.Chunk == nil {
return false
}
downloaded := conn.Chunk.Downloaded
return downloaded > 0 && downloaded < f.meta.Res.Size
return downloaded > 0 && downloaded < f.knownSize()
}
// canProbeSequentialResumeLocked reports whether the prefix can be resumed
@@ -2226,7 +2194,7 @@ func (f *Fetcher) onDownloadComplete() {
needsMoreData := false
if f.meta.Res.Range {
needsMoreData = conn.Chunk != nil && (conn.Chunk.openEnded() || conn.Chunk.remain() > 0)
} else if f.meta.Res.Size > 0 {
} else if f.hasKnownSize() && f.meta.Res.Size > 0 {
needsMoreData = conn.Downloaded < f.meta.Res.Size
} else {
needsMoreData = !conn.Completed && conn.State != connCompleted
@@ -2236,7 +2204,7 @@ func (f *Fetcher) onDownloadComplete() {
// This connection has remaining work and isn't done
// Check if it failed with 403 (server limit) - these can be ignored if other connections completed the work
if conn.State == connFailed && conn.failed {
if re := extractRequestError(conn.lastErr); re != nil && re.Code == 403 {
if re := extractRequestError(conn.lastErr); re != nil && re.Code == 403 && f.hasKnownSize() {
// 403 is server connection limit, check if other connections will complete this chunk
continue
}
@@ -2247,7 +2215,7 @@ func (f *Fetcher) onDownloadComplete() {
}
// If total downloaded matches file size, consider it a success regardless of connection failures
downloadComplete := f.meta.Res.Size > 0 && totalDownloaded >= f.meta.Res.Size
downloadComplete := f.hasKnownSize() && f.meta.Res.Size > 0 && totalDownloaded >= f.meta.Res.Size
// Integrity errors must win over byte-count completion. A malformed response
// may be detected only after the expected bytes have already been written.
@@ -2264,7 +2232,7 @@ func (f *Fetcher) onDownloadComplete() {
for _, conn := range f.connections {
if conn.State == connFailed && conn.failed {
// Skip 403 errors (server connection limit) - these are expected when exceeding server's limit
if re := extractRequestError(conn.lastErr); re != nil && re.Code == 403 {
if re := extractRequestError(conn.lastErr); re != nil && re.Code == 403 && f.hasKnownSize() {
continue
}
if re := extractRequestError(conn.lastErr); re != nil {
@@ -2364,7 +2332,7 @@ func (f *Fetcher) checkCompletion() bool {
totalDownloaded += conn.Downloaded
}
if f.meta.Res.Size > 0 && totalDownloaded >= f.meta.Res.Size {
if f.hasKnownSize() && f.meta.Res.Size > 0 && totalDownloaded >= f.meta.Res.Size {
// Don't start a new goroutine - let the caller handle completion
return true
}
@@ -2391,6 +2359,14 @@ func (f *Fetcher) checkCompletion() bool {
// Patch modifies the HTTP request information.
func (f *Fetcher) Patch(req *base.Request, opts *base.Options) error {
if opts != nil && opts.Size > 0 {
f.connMu.Lock()
f.meta.Opts.Size = opts.Size
if f.meta.Res != nil && !f.hasKnownSize() {
f.setResourceSizeLocked(0, false)
}
f.connMu.Unlock()
}
// Patch request info
if req != nil {
if req.URL != "" {
@@ -2488,8 +2464,8 @@ func (f *Fetcher) Pause() error {
conn := f.connections[0]
if conn != nil && conn.ID == 0 && conn.Chunk != nil && !conn.Completed {
conn.Chunk.Begin = prefetched
if f.meta.Res.Size > 0 {
conn.Chunk.setEnd(f.meta.Res.Size - 1)
if f.knownSize() > 0 {
conn.Chunk.setEnd(f.knownSize() - 1)
} else {
conn.Chunk.End = nil
}
+10 -3
View File
@@ -19,10 +19,11 @@ type fetcherData struct {
IfRange string // Strong ETag or Last-Modified validator for safe resume
RangeReprobeEligible bool // Origin advertised Range before falling back to sequential mode
RangeValidatorPinned bool // Recovered Range mode must keep the same If-Range validator
SequentialSizeUnknown bool // Interrupted chunked restart must retry from byte zero
SequentialSizeUnknown bool // Interrupted chunked sequential streams restart from byte zero
SizeKnown *bool // Nil for legacy snapshots; false means Size is only a display estimate
Range *bool // Authoritative Range mode; nil for records saved by older versions
ResourceSize *int64 // Authoritative resource size from the same snapshot as Connections
FileSize *int64 // Authoritative single-file size; nil for older records
ResourceSize *int64 // Display size from the same snapshot; authority is recorded by SizeKnown
FileSize *int64 // Single-file size from the same snapshot; nil for older records
}
// ============================================================================
@@ -93,6 +94,7 @@ func (fm *FetcherManager) Store(f fetcher.Fetcher) (data any, err error) {
rangeReprobeEligible := _f.rangeReprobeEligible
rangeValidatorPinned := _f.rangeValidatorPinned
sequentialSizeUnknown := _f.sequentialSizeUnknown
sizeKnown := _f.hasKnownSize()
var rangeMode *bool
var resourceSize *int64
var fileSize *int64
@@ -115,6 +117,7 @@ func (fm *FetcherManager) Store(f fetcher.Fetcher) (data any, err error) {
RangeReprobeEligible: rangeReprobeEligible,
RangeValidatorPinned: rangeValidatorPinned,
SequentialSizeUnknown: sequentialSizeUnknown,
SizeKnown: &sizeKnown,
Range: rangeMode,
ResourceSize: resourceSize,
FileSize: fileSize,
@@ -168,6 +171,10 @@ func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v
fetcher.rangeReprobeEligible = fd.RangeReprobeEligible
fetcher.rangeValidatorPinned = fd.RangeValidatorPinned
fetcher.sequentialSizeUnknown = fd.SequentialSizeUnknown
if fd.SizeKnown != nil {
known := *fd.SizeKnown
fetcher.sizeKnown = &known
}
if fd.Range != nil && fetcher.meta.Res != nil {
// Recovery-critical resource fields live in the same persisted snapshot
// as Connections, making them authoritative over task metadata that may
+9 -1
View File
@@ -1287,6 +1287,12 @@ func TestFetcher_ValidRangeTakesOverResolveResponse(t *testing.T) {
}
func TestFetcher_UnknownSizeRangeUsesOpenEndedWorker(t *testing.T) {
for _, hint := range []int64{0, 3, 8 * 1024 * 1024} {
t.Run(fmt.Sprint(hint), func(t *testing.T) { testFetcherUnknownSizeRange(t, hint) })
}
}
func testFetcherUnknownSizeRange(t *testing.T, hint int64) {
const resolveETag = `"unknown-size-v1"`
payload := make([]byte, 4*1024*1024)
for i := range payload {
@@ -1352,6 +1358,7 @@ func TestFetcher_UnknownSizeRangeUsesOpenEndedWorker(t *testing.T) {
f := buildFetcher()
f.Meta().Opts = &base.Options{
Size: hint,
Path: t.TempDir(),
Name: "unknown-size-range.data",
Extra: &http.OptsExtra{
@@ -1363,7 +1370,7 @@ func TestFetcher_UnknownSizeRangeUsesOpenEndedWorker(t *testing.T) {
}
defer f.Close()
if f.meta.Res.Size != 0 || !f.meta.Res.Range {
if f.meta.Res.Size != hint || f.hasKnownSize() || !f.meta.Res.Range {
t.Fatalf("resolved resource = %+v, want unknown size with Range support", f.meta.Res)
}
if err := f.Start(); err != nil {
@@ -1595,6 +1602,7 @@ func TestFetcher_UnknownSizeRangeRemainsOpenEndedAfterPause(t *testing.T) {
f := buildFetcher()
f.Meta().Opts = &base.Options{
Size: 3,
Path: t.TempDir(),
Name: "pause-open-range.data",
Extra: &http.OptsExtra{
+245
View File
@@ -0,0 +1,245 @@
package http
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"sync/atomic"
"testing"
"github.com/GopeedLab/gopeed/pkg/base"
)
func TestFetcher_OptionsSizeHintAndPatch(t *testing.T) {
for _, known := range []bool{false, true} {
for _, payload := range []string{"", "0123456789"} {
t.Run(fmt.Sprintf("known=%v/bytes=%d", known, len(payload)), func(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
if known {
w.Header().Set("Content-Length", fmt.Sprint(len(payload)))
}
w.(http.Flusher).Flush()
fmt.Fprint(w, payload)
}))
defer server.Close()
f := buildFetcher()
defer f.Close()
f.Meta().Opts = &base.Options{Size: 7, Path: t.TempDir(), Name: "output.bin"}
if err := f.Resolve(&base.Request{URL: server.URL}); err != nil {
t.Fatal(err)
}
want := int64(7)
if known {
want = int64(len(payload))
}
if f.meta.Res.Size != want || f.hasKnownSize() != known {
t.Fatalf("resolved size=%d known=%v", f.meta.Res.Size, f.hasKnownSize())
}
if err := f.Patch(nil, &base.Options{Size: 9}); err != nil {
t.Fatal(err)
}
if !known {
want = 9
}
if f.meta.Res.Size != want {
t.Fatal("hint update overrode a real size or failed to update an estimate")
}
if err := f.Start(); err != nil {
t.Fatal(err)
}
if err := f.Wait(); err != nil {
t.Fatal(err)
}
if f.meta.Res.Size != int64(len(payload)) || !f.hasKnownSize() {
t.Fatalf("final size=%d known=%v", f.meta.Res.Size, f.hasKnownSize())
}
snapshot, err := new(FetcherManager).Store(f)
if err != nil {
t.Fatal(err)
}
fd := snapshot.(*fetcherData)
if fd.SizeKnown == nil || !*fd.SizeKnown {
t.Fatal("snapshot lost confirmed size authority")
}
_, restore := new(FetcherManager).Restore()
restored := restore(f.meta, fd).(*Fetcher)
if !restored.hasKnownSize() || restored.knownSize() != int64(len(payload)) {
t.Fatal("restored size lost authority, including known zero")
}
})
}
}
}
func TestFetcher_SizeHintCannotCompleteForbiddenUnknownRange(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Range") != "" {
w.WriteHeader(http.StatusForbidden)
return
}
w.Header().Set("Accept-Ranges", "bytes")
w.(http.Flusher).Flush()
fmt.Fprint(w, "0123456789")
w.(http.Flusher).Flush()
<-r.Context().Done()
}))
defer server.Close()
f := buildFetcher()
defer f.Close()
f.Meta().Opts = &base.Options{Size: 3, Path: t.TempDir(), Name: "output.bin"}
if err := f.Resolve(&base.Request{URL: server.URL}); err != nil {
t.Fatal(err)
}
if err := f.Start(); err != nil {
t.Fatal(err)
}
if err := f.Wait(); err == nil {
t.Fatal("a failed unknown-length Range request was completed from an estimate")
}
}
func TestFetcher_SizeHintPreservesHTTPIntegrity(t *testing.T) {
for _, chunked := range []bool{false, true} {
for _, broken := range []bool{false, true} {
t.Run(fmt.Sprintf("chunked=%v/broken=%v", chunked, broken), func(t *testing.T) {
payload := []byte("0123456789")
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
if chunked && broken {
conn, rw, err := w.(http.Hijacker).Hijack()
if err != nil {
t.Error(err)
return
}
defer conn.Close()
fmt.Fprint(rw, "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\na\r\n0123456789\r\n")
rw.Flush() // Omit the final chunk to simulate a connection interruption.
return
}
if !chunked {
length := len(payload)
if broken {
length++
}
w.Header().Set("Content-Length", fmt.Sprint(length))
}
w.(http.Flusher).Flush()
w.Write(payload)
}))
defer server.Close()
f := buildFetcher()
defer f.Close()
f.Meta().Opts = &base.Options{Size: 3, Path: t.TempDir(), Name: "output.bin"}
if err := f.Resolve(&base.Request{URL: server.URL}); err != nil {
t.Fatal(err)
}
if !chunked {
length := int64(len(payload))
if broken {
length++
}
if f.meta.Res.Size != length || f.sequentialSizeUnknown {
t.Fatal("HTTP length did not override size hint")
}
} else if f.meta.Res.Size != 3 || !f.sequentialSizeUnknown {
t.Fatal("unknown HTTP length did not retain the display hint")
}
if err := f.Start(); err != nil {
t.Fatal(err)
}
err := f.Wait()
if broken {
if err == nil {
t.Fatal("truncated HTTP response succeeded after exceeding the size hint")
}
return
}
if err != nil {
t.Fatal(err)
}
if f.meta.Res.Size != int64(len(payload)) || f.sequentialSizeUnknown {
t.Fatalf("completed size=%d unknown=%v", f.meta.Res.Size, f.sequentialSizeUnknown)
}
})
}
}
}
func TestFetcher_SizeHintSurvivesRestore(t *testing.T) {
for _, partial := range []bool{false, true} {
for _, fixedLength := range []bool{false, true} {
t.Run(fmt.Sprintf("partial=%v/fixedLength=%v", partial, fixedLength), func(t *testing.T) {
payload := []byte("0123456789")
var requests atomic.Int64
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
request := requests.Add(1)
if got := r.Header.Get("Range"); got != "" {
t.Errorf("estimated sizes must not be used for resume ranges: %s", got)
}
if fixedLength && request > 1 {
w.Header().Set("Content-Length", fmt.Sprint(len(payload)))
}
w.(http.Flusher).Flush()
if fixedLength && request == 2 {
w.Write(payload[:4]) // Retry must discard this interrupted attempt.
return
}
w.Write(payload)
}))
defer server.Close()
f := buildFetcher()
opts := &base.Options{Size: 3, Path: t.TempDir(), Name: "output.bin"}
f.Meta().Opts = opts
if err := f.Resolve(&base.Request{URL: server.URL}); err != nil {
t.Fatal(err)
}
fm := new(FetcherManager)
snapshot, err := fm.Store(f)
if err != nil {
t.Fatal(err)
}
data, err := json.Marshal(snapshot)
if err != nil {
t.Fatal(err)
}
var saved fetcherData
if err := json.Unmarshal(data, &saved); err != nil {
t.Fatal(err)
}
if !saved.SequentialSizeUnknown || saved.SizeKnown == nil || *saved.SizeKnown || *saved.ResourceSize != 3 {
t.Fatal("snapshot lost the size hint or its advisory state")
}
f.Close()
// An interrupted earlier attempt left a tail longer than the new output.
if err := os.WriteFile(filepath.Join(opts.Path, opts.Name), bytes.Repeat([]byte("old"), 100), 0600); err != nil {
t.Fatal(err)
}
if partial {
chunk := newChunk(0, 0)
chunk.Downloaded = 7 // Already exceeds the display estimate.
saved.Connections = []*connection{{ID: 0, Role: rolePrimary, State: connNotStarted, Chunk: chunk, Downloaded: 7}}
}
_, restore := fm.Restore()
restored := restore(f.meta, &saved).(*Fetcher)
restored.Setup(f.ctl)
defer restored.Close()
if err := restored.Start(); err != nil {
t.Fatal(err)
}
if err := restored.Wait(); err != nil {
t.Fatal(err)
}
body, err := os.ReadFile(restored.meta.SingleFilepath())
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(body, payload) || restored.meta.Res.Size != int64(len(payload)) || restored.Progress().TotalDownloaded() != int64(len(payload)) {
t.Fatalf("restored stream retained an estimate or stale bytes: body=%q size=%d progress=%d", body, restored.meta.Res.Size, restored.Progress().TotalDownloaded())
}
})
}
}
}
+3
View File
@@ -108,6 +108,9 @@ type FileInfo struct {
// Options for download
type Options struct {
// Size is an optional HTTP download size estimate. HTTP metadata and clean EOF take precedence.
// Other protocols ignore it.
Size int64 `json:"size,omitempty"`
// Download file name
Name string `json:"name"`
// Download file path
+31 -4
View File
@@ -607,7 +607,7 @@ func (d *Downloader) createDirect(req *base.Request, opts *base.Options) (taskId
return
}
fetcher.Meta().Req = req
initOpt, err := d.initOptions(opts)
initOpt, err := d.initOptions(opts, req.URL)
if err != nil {
return
}
@@ -654,7 +654,7 @@ func (d *Downloader) CreateWithOptions(rrId string, opts *base.Options) (taskId
if opts == nil {
opts = &base.Options{}
}
opts, err = d.initOptions(opts.Clone())
opts, err = d.initOptions(opts.Clone(), fetcher.Meta().Req.URL)
if err != nil {
return "", err
}
@@ -670,6 +670,13 @@ func (d *Downloader) CreateWithOptions(rrId string, opts *base.Options) (taskId
if res := fetcher.Meta().Res; res != nil {
res.CalcSize(fetcher.Meta().Opts.SelectFiles)
}
// Resolve has no creation options. Publish a submitted HTTP estimate now
// while preserving any size already confirmed by the response.
if opts.Size > 0 {
if err := fetcher.Patch(nil, &base.Options{Size: opts.Size}); err != nil {
return "", err
}
}
defer func() {
d.fetcherMapLock.Lock()
delete(d.fetcherCache, rrId)
@@ -712,7 +719,7 @@ func (d *Downloader) Patch(id string, req *base.Request, opts *base.Options) err
}
// Call the fetcher's Patch method
if err := task.fetcher.Patch(req, opts); err != nil {
if err := task.fetcher.Patch(req, httpSizeOptions(opts, task.Protocol)); err != nil {
return err
}
@@ -1123,6 +1130,9 @@ func (d *Downloader) RuntimeStatus(id string) (*TaskRuntimeStatus, error) {
result.ExtractProgress = task.Progress.ExtractProgress
}
if task.Meta == nil || task.Meta.Res == nil {
if task.Meta != nil && task.Meta.Opts != nil && task.Protocol == "http" {
result.Total = max(0, task.Meta.Opts.Size)
}
return result, nil
}
@@ -1803,10 +1813,27 @@ func (d *Downloader) doCreate(f fetcher.Fetcher, opts *base.Options) (taskId str
return
}
func (d *Downloader) initOptions(opts *base.Options) (*base.Options, error) {
// httpSizeOptions keeps estimates out of other protocols, including HLS URLs
// served over HTTP. Clone before filtering because batch tasks may share options.
func httpSizeOptions(opts *base.Options, protocol string) *base.Options {
if opts != nil && opts.Size != 0 && (protocol != "http" || opts.Size < 0) {
opts = opts.Clone()
opts.Size = 0
}
return opts
}
func (d *Downloader) initOptions(opts *base.Options, requestURL string) (*base.Options, error) {
if opts == nil {
opts = &base.Options{}
}
if opts.Size != 0 {
fm, err := d.parseFm(requestURL)
if err != nil {
return nil, err
}
opts = httpSizeOptions(opts, fm.Name())
}
if opts.SelectFiles == nil {
opts.SelectFiles = make([]int, 0)
}
+17 -1
View File
@@ -30,6 +30,21 @@ support must remain disabled. Do not declare the sum of the input sizes as the
output size. The downloader reads the merged output as one file; FFmpeg errors
propagate through the stream instead of completing a partial download.
Extension resolve results can set `files[].size` to an estimated output size,
for example the sum of the selected video and audio input lengths. Clients forward
this value as the optional task creation `opts.size` for HTTP downloads only
(including Blob URLs served over HTTP). Other protocols, including HLS, ignore
this option. The estimate is persisted
and displayed while queued and downloading. A valid HTTP total length (including
zero) always overrides it; otherwise a clean EOF records the actual output length.
Estimates never determine byte ranges, file allocation, integrity checks, or task
completion. Size authority is persisted alongside HTTP connection state for
pause/retry and process restart.
Blob options are unchanged: `createObjectURL`'s `size` is an exact source length,
not an estimate. Leave it unset for generated merge output and pass any estimate
through the extension resolve file instead.
## JS inputs, including SABR
Stream inputs **must** be created by an `inputs({ signal })` callback. It returns
@@ -130,7 +145,8 @@ associates its input progress with that Blob. No public progress callback or
extra `createObjectURL` argument is required. Return the merge stream itself;
wrapping it in another stream loses this association. Downloaded bytes count
unique input offsets; speed counts received input bytes, including re-fetches.
The task stays downloading with unknown total size while queued or merging;
The task stays downloading while queued or merging, displaying the optional
`files[].size` estimate (or an unknown total when omitted);
once input traffic stops the displayed speed settles to zero. Completion uses
the actual merged output size. Generated output waits are governed by producer
cancellation/errors rather than the HTTP body idle timeout; HTTP media inputs
+158
View File
@@ -0,0 +1,158 @@
package download
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"testing"
"time"
"github.com/GopeedLab/gopeed/pkg/base"
)
func TestExtensionResolveSizeThroughTaskCreation(t *testing.T) {
for _, mode := range []string{"http", "blob"} {
for _, known := range []bool{false, true} {
t.Run(fmt.Sprintf("%s/known=%v", mode, known), func(t *testing.T) {
payload := []byte("0123456789")
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body := payload
if r.URL.Path == "/empty" {
body = nil
}
if known {
w.Header().Set("Content-Length", fmt.Sprint(len(body)))
}
w.(http.Flusher).Flush()
w.Write(body)
}))
defer server.Close()
d := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir(), TempDir: t.TempDir()})
if err := d.Setup(); err != nil {
t.Fatal(err)
}
defer d.Clear()
d.cfg.MaxRunning = 0 // Inspect persisted queued tasks before HTTP resolution.
extDir := t.TempDir()
manifest := `{"name":"size-hint","author":"test","title":"Size hint","version":"1.0.0","scripts":[{"event":"onResolve","match":{"urls":["https://example.com/size-hint"]},"entry":"index.js"}]}`
script := fmt.Sprintf(`gopeed.events.onResolve(async ctx=>{
const mode=%q, origin=%q;
const file=async(name,size,empty)=>({name,size,req:{url:mode==='http'?origin+(empty?'/empty':'/data'):
await gopeed.runtime.blob.createObjectURL(()=>new ReadableStream({start(c){if(!empty)c.enqueue(new Uint8Array([48,49,50,51,52,53,54,55,56,57]));c.close();}}))}});
ctx.res={files:[await file('under.bin',3,false),await file('over.bin',1048576,false),await file('empty.bin',99,true)]};
});`, mode, server.URL)
for name, contents := range map[string]string{"manifest.json": manifest, "index.js": script} {
if err := os.WriteFile(filepath.Join(extDir, name), []byte(contents), 0600); err != nil {
t.Fatal(err)
}
}
if _, err := d.InstallExtensionByFolder(extDir, true); err != nil {
t.Fatal(err)
}
rr, err := d.Resolve(&base.Request{URL: "https://example.com/size-hint"})
if err != nil {
t.Fatal(err)
}
output := t.TempDir()
batch := &base.CreateTaskBatch{}
for _, file := range rr.Res.Files {
batch.Reqs = append(batch.Reqs, &base.CreateTaskBatchItem{Req: file.Req, Opts: &base.Options{Size: file.Size, Name: file.Name, Path: output}})
}
wire, err := json.Marshal(batch)
if err != nil {
t.Fatal(err)
}
var submitted base.CreateTaskBatch
if err := json.Unmarshal(wire, &submitted); err != nil {
t.Fatal(err)
}
ids, err := d.CreateDirectBatch(&submitted)
if err != nil {
t.Fatal(err)
}
for i, id := range ids {
task := d.GetTask(id)
if task.Meta.Res != nil {
t.Fatal("a size estimate must not mark the task resolved")
}
status, err := d.RuntimeStatus(id)
if err != nil || status.Total != rr.Res.Files[i].Size {
t.Fatalf("queued size lost: status=%+v err=%v", status, err)
}
var saved Task
if _, err := d.storage.Get(bucketTask, id, &saved); err != nil || saved.Meta.Opts.Size != status.Total {
t.Fatal("queued estimate was not persisted", err)
}
if err := d.doStart(task); err != nil {
t.Fatal(err)
}
waitForTaskTerminal(t, d, id, 5*time.Second)
body, err := os.ReadFile(filepath.Join(output, rr.Res.Files[i].Name))
if err != nil {
t.Fatal(err)
}
want := payload
if i == 2 {
want = nil
}
if !bytes.Equal(body, want) {
t.Fatalf("estimate changed output: %q", body)
}
status, err = d.RuntimeStatus(id)
if err != nil || status.Total != int64(len(want)) || status.Downloaded != int64(len(want)) || task.Meta.Res.Files[0].Size != int64(len(want)) {
t.Fatalf("final size not updated: %+v err=%v", status, err)
}
}
})
}
}
}
func TestResolvedHTTPSizeEstimateAtCreation(t *testing.T) {
for _, known := range []bool{false, true} {
t.Run(fmt.Sprintf("known=%v", known), func(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
if known {
w.Header().Set("Content-Length", "10")
}
w.(http.Flusher).Flush()
w.Write([]byte("0123456789"))
}))
defer server.Close()
d := NewDownloader(&DownloaderConfig{StorageDir: t.TempDir(), TempDir: t.TempDir()})
if err := d.Setup(); err != nil {
t.Fatal(err)
}
defer d.Clear()
d.cfg.MaxRunning = 0
rr, err := d.Resolve(&base.Request{URL: server.URL})
if err != nil {
t.Fatal(err)
}
id, err := d.CreateWithOptions(rr.ID, &base.Options{Size: 3, Path: t.TempDir(), Name: "output.bin"})
if err != nil {
t.Fatal(err)
}
want := int64(3)
if known {
want = 10
}
status, err := d.RuntimeStatus(id)
if err != nil || status.Total != want || d.GetTask(id).Meta.Res.Files[0].Size != want {
t.Fatalf("resolve-before-create size=%+v want=%d err=%v", status, want, err)
}
if err := d.doStart(d.GetTask(id)); err != nil {
t.Fatal(err)
}
waitForTaskTerminal(t, d, id, 5*time.Second)
task := d.GetTask(id)
if task.Status != base.DownloadStatusDone || task.Meta.Res.Size != 10 {
t.Fatalf("estimate changed completion: status=%s size=%d", task.Status, task.Meta.Res.Size)
}
})
}
}
+212
View File
@@ -0,0 +1,212 @@
package download
import (
"bytes"
"fmt"
"os"
"path/filepath"
"testing"
"time"
"github.com/GopeedLab/gopeed/pkg/base"
)
func TestExtensionFileSizeIsAdvisory(t *testing.T) {
video, err := os.ReadFile("engine/testdata/ffmpeg/video.mp4")
if err != nil {
t.Fatal(err)
}
audio, err := os.ReadFile("engine/testdata/ffmpeg/audio.mp4")
if err != nil {
t.Fatal(err)
}
for _, mode := range []string{"plain", "merge", "wrapped-merge"} {
var expected []byte
for _, size := range []int64{0, 3, 1 << 20} {
t.Run(fmt.Sprintf("%s/size=%d", mode, size), func(t *testing.T) {
d := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir(), TempDir: t.TempDir(), RefreshInterval: 50})
if err := d.Setup(); err != nil {
t.Fatal(err)
}
defer d.Clear()
e, _ := d.newExtensionEngine()
defer e.Close()
e.Runtime.Set("videoBytes", e.Runtime.NewArrayBuffer(video))
e.Runtime.Set("audioBytes", e.Runtime.NewArrayBuffer(audio))
e.Runtime.Set("mode", mode)
result, err := e.RunString(`__gopeed_blob_create_object_url(()=>{
const source=b=>new ReadableStream({start(c){c.enqueue(new Uint8Array(b));c.close();}});
if (mode==='plain') return source([0,1,2,3,4,5,6,7,8,9]);
const output=__gopeed_ffmpeg.merge({inputs:()=>({video:source(videoBytes),audio:source(audioBytes)})});
if (mode==='merge') return output;
const reader=output.getReader();
return new ReadableStream({async pull(c){const {done,value}=await reader.read();if(done)c.close();else c.enqueue(value);},cancel(r){return reader.cancel(r);}});
},{contentType:'video/mp4',range:false})`)
if err != nil {
t.Fatal(err)
}
output := t.TempDir()
id, err := d.CreateDirect(&base.Request{URL: fmt.Sprint(result)}, &base.Options{Size: size, Path: output, Name: "output.bin"})
if err != nil {
t.Fatal(err)
}
waitForTaskTerminal(t, d, id, 30*time.Second)
body, err := os.ReadFile(filepath.Join(output, "output.bin"))
if err != nil {
t.Fatal(err)
}
if size == 0 {
expected = body
if mode == "plain" && !bytes.Equal(body, []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) {
t.Fatalf("unexpected stream output: %v", body)
}
if mode != "plain" && !bytes.Contains(body, []byte("moof")) {
t.Fatal("invalid merged output")
}
} else if !bytes.Equal(body, expected) {
t.Fatalf("size hint changed output: got %d bytes, want %d", len(body), len(expected))
}
task := d.GetTask(id)
task.lock.Lock()
defer task.lock.Unlock()
if task.Meta.Res.Size != int64(len(body)) || task.Meta.Res.Files[0].Size != int64(len(body)) {
t.Fatalf("final metadata must use actual size: %#v", task.Meta.Res)
}
})
}
}
}
func TestExtensionFileSizeYieldsToBlobContentLength(t *testing.T) {
for _, materialized := range []bool{false, true} {
for _, size := range []int64{3, 1 << 20} {
t.Run(fmt.Sprintf("materialized=%v/estimate=%d", materialized, size), func(t *testing.T) {
d := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir(), TempDir: t.TempDir()})
if err := d.Setup(); err != nil {
t.Fatal(err)
}
defer d.Clear()
e, _ := d.newExtensionEngine()
defer e.Close()
e.Runtime.Set("materialized", materialized)
result, err := e.RunString(`(()=>{
const payload=new Uint8Array([0,1,2,3,4,5,6,7,8,9]);
const source=materialized?new Blob([payload]):()=>new ReadableStream({start(c){c.enqueue(payload);c.close();}});
return __gopeed_blob_create_object_url(source,{size:10});
})()`)
if err != nil {
t.Fatal(err)
}
output := t.TempDir()
id, err := d.CreateDirect(&base.Request{URL: fmt.Sprint(result)}, &base.Options{Size: size, Path: output, Name: "output.bin"})
if err != nil {
t.Fatal(err)
}
waitForTaskTerminal(t, d, id, 5*time.Second)
body, err := os.ReadFile(filepath.Join(output, "output.bin"))
if err != nil || !bytes.Equal(body, []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) {
t.Fatalf("unexpected body=%v err=%v", body, err)
}
task := d.GetTask(id)
if task.Meta.Res.Size != 10 || task.Meta.Res.Files[0].Size != 10 {
t.Fatalf("Blob Content-Length did not override estimate: %+v", task.Meta.Res)
}
})
}
}
}
func TestExtensionFileSizeHintWaitsForEOF(t *testing.T) {
for _, fail := range []bool{false, true} {
for _, size := range []int64{3, 1 << 20} {
t.Run(fmt.Sprintf("fail=%v/size=%d", fail, size), func(t *testing.T) {
d := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir(), TempDir: t.TempDir(), RefreshInterval: 50})
if err := d.Setup(); err != nil {
t.Fatal(err)
}
defer d.Clear()
e, _ := d.newExtensionEngine()
defer e.Close()
e.Runtime.Set("fail", fail)
result, err := e.RunString(`__gopeed_blob_create_object_url(()=>new ReadableStream({
start(c){c.enqueue(new Uint8Array([0,1,2,3,4,5,6,7,8,9]));globalThis.finish=()=>{if(fail)c.error(new Error('upstream failed'));else c.close();};}
}),{})`)
if err != nil {
t.Fatal(err)
}
output := t.TempDir()
id, err := d.CreateDirect(&base.Request{URL: fmt.Sprint(result)}, &base.Options{Size: size, Path: output, Name: "output.bin"})
if err != nil {
t.Fatal(err)
}
waitForTaskDownloadedBetween(t, d, id, 10, 10, 5*time.Second)
task := d.GetTask(id)
task.lock.Lock()
if task.Meta.Res.Size != size || d.taskStatus(task) != base.DownloadStatusRunning {
t.Errorf("hint must remain visible until EOF: size=%d status=%s", task.Meta.Res.Size, d.taskStatus(task))
}
task.lock.Unlock()
stat, err := os.Stat(filepath.Join(output, "output.bin"))
if err != nil || stat.Size() != 10 {
t.Fatalf("hint must not preallocate or truncate output: stat=%v err=%v", stat, err)
}
if _, err := e.RunString(`finish()`); err != nil {
t.Fatal(err)
}
if fail {
waitForTaskStatus(t, d, id, base.DownloadStatusError, 5*time.Second)
} else {
waitForTaskTerminal(t, d, id, 5*time.Second)
}
})
}
}
}
func TestExtensionFileSizeHintPauseContinue(t *testing.T) {
for _, size := range []int64{3, 1 << 20} {
t.Run(fmt.Sprint(size), func(t *testing.T) {
d := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir(), TempDir: t.TempDir(), RefreshInterval: 50})
if err := d.Setup(); err != nil {
t.Fatal(err)
}
defer d.Clear()
e, _ := d.newExtensionEngine()
defer e.Close()
result, err := e.RunString(`(()=>{
let opens=0;
return __gopeed_blob_create_object_url(()=>new ReadableStream({start(c){
if (++opens===1) c.enqueue(new Uint8Array([0,1,2,3,4,5,6,7,8,9]));
else {c.enqueue(new Uint8Array([5,6]));c.close();}
}}),{});
})()`)
if err != nil {
t.Fatal(err)
}
output := t.TempDir()
id, err := d.CreateDirect(&base.Request{URL: fmt.Sprint(result)}, &base.Options{Size: size, Path: output, Name: "output.bin"})
if err != nil {
t.Fatal(err)
}
waitForTaskDownloadedBetween(t, d, id, 10, 10, 5*time.Second)
filter := &TaskFilter{IDs: []string{id}}
if err := d.Pause(filter); err != nil {
t.Fatal(err)
}
if err := d.Continue(filter); err != nil {
t.Fatal(err)
}
waitForTaskTerminal(t, d, id, 5*time.Second)
body, err := os.ReadFile(filepath.Join(output, "output.bin"))
if err != nil {
t.Fatal(err)
}
task := d.GetTask(id)
task.lock.Lock()
defer task.lock.Unlock()
if !bytes.Equal(body, []byte{5, 6}) || task.Meta.Res.Size != 2 {
t.Fatalf("resumed stream kept an estimate or stale bytes: body=%v size=%d", body, task.Meta.Res.Size)
}
})
}
}
+12 -6
View File
@@ -4,13 +4,22 @@ part 'options.g.dart';
@JsonSerializable(explicitToJson: true)
class Options {
/// HTTP-only size estimate, superseded by HTTP metadata or completion.
int size;
String name;
String path;
bool asDefaultPath;
List<int> selectFiles;
Object? extra;
Options({this.name = '', this.path = '', this.asDefaultPath = false, this.selectFiles = const [], this.extra});
Options({
this.size = 0,
this.name = '',
this.path = '',
this.asDefaultPath = false,
this.selectFiles = const [],
this.extra,
});
factory Options.fromJson(Map<String, dynamic> json) => _$OptionsFromJson(json);
@@ -22,17 +31,14 @@ class ChecksumOption {
String algorithm;
String expected;
ChecksumOption({
this.algorithm = '',
this.expected = '',
});
ChecksumOption({this.algorithm = '', this.expected = ''});
factory ChecksumOption.fromJson(Map<String, dynamic> json) => _$ChecksumOptionFromJson(json);
Map<String, dynamic> toJson() => _$ChecksumOptionToJson(this);
}
@JsonSerializable()
@JsonSerializable(explicitToJson: true)
class OptsExtraHttp {
int connections;
bool? autoTorrent;
+39 -29
View File
@@ -7,14 +7,20 @@ part of 'options.dart';
// **************************************************************************
Options _$OptionsFromJson(Map<String, dynamic> json) => Options(
size: (json['size'] as num?)?.toInt() ?? 0,
name: json['name'] as String? ?? '',
path: json['path'] as String? ?? '',
asDefaultPath: json['asDefaultPath'] as bool? ?? false,
selectFiles: (json['selectFiles'] as List<dynamic>?)?.map((e) => (e as num).toInt()).toList() ?? const [],
selectFiles:
(json['selectFiles'] as List<dynamic>?)
?.map((e) => (e as num).toInt())
.toList() ??
const [],
extra: json['extra'],
);
Map<String, dynamic> _$OptionsToJson(Options instance) => <String, dynamic>{
'size': instance.size,
'name': instance.name,
'path': instance.path,
'asDefaultPath': instance.asDefaultPath,
@@ -22,34 +28,38 @@ Map<String, dynamic> _$OptionsToJson(Options instance) => <String, dynamic>{
'extra': ?instance.extra,
};
ChecksumOption _$ChecksumOptionFromJson(Map<String, dynamic> json) => ChecksumOption(
algorithm: json['algorithm'] as String? ?? '',
expected: json['expected'] as String? ?? '',
);
ChecksumOption _$ChecksumOptionFromJson(Map<String, dynamic> json) =>
ChecksumOption(
algorithm: json['algorithm'] as String? ?? '',
expected: json['expected'] as String? ?? '',
);
Map<String, dynamic> _$ChecksumOptionToJson(ChecksumOption instance) => <String, dynamic>{
'algorithm': instance.algorithm,
'expected': instance.expected,
};
Map<String, dynamic> _$ChecksumOptionToJson(ChecksumOption instance) =>
<String, dynamic>{
'algorithm': instance.algorithm,
'expected': instance.expected,
};
OptsExtraHttp _$OptsExtraHttpFromJson(Map<String, dynamic> json) => OptsExtraHttp(
connections: (json['connections'] as num?)?.toInt() ?? 0,
autoTorrent: json['autoTorrent'] as bool?,
deleteTorrentAfterDownload: json['deleteTorrentAfterDownload'] as bool?,
autoExtract: json['autoExtract'] as bool?,
archivePassword: json['archivePassword'] as String? ?? '',
deleteAfterExtract: json['deleteAfterExtract'] as bool? ?? false,
checksum: json['checksum'] == null
? null
: ChecksumOption.fromJson(json['checksum'] as Map<String, dynamic>),
);
OptsExtraHttp _$OptsExtraHttpFromJson(Map<String, dynamic> json) =>
OptsExtraHttp(
connections: (json['connections'] as num?)?.toInt() ?? 0,
autoTorrent: json['autoTorrent'] as bool?,
deleteTorrentAfterDownload: json['deleteTorrentAfterDownload'] as bool?,
autoExtract: json['autoExtract'] as bool?,
archivePassword: json['archivePassword'] as String? ?? '',
deleteAfterExtract: json['deleteAfterExtract'] as bool? ?? false,
checksum: json['checksum'] == null
? null
: ChecksumOption.fromJson(json['checksum'] as Map<String, dynamic>),
);
Map<String, dynamic> _$OptsExtraHttpToJson(OptsExtraHttp instance) => <String, dynamic>{
'connections': instance.connections,
'autoTorrent': ?instance.autoTorrent,
'deleteTorrentAfterDownload': ?instance.deleteTorrentAfterDownload,
'autoExtract': ?instance.autoExtract,
'archivePassword': instance.archivePassword,
'deleteAfterExtract': instance.deleteAfterExtract,
'checksum': ?instance.checksum?.toJson(),
};
Map<String, dynamic> _$OptsExtraHttpToJson(OptsExtraHttp instance) =>
<String, dynamic>{
'connections': instance.connections,
'autoTorrent': ?instance.autoTorrent,
'deleteTorrentAfterDownload': ?instance.deleteTorrentAfterDownload,
'autoExtract': ?instance.autoExtract,
'archivePassword': instance.archivePassword,
'deleteAfterExtract': instance.deleteAfterExtract,
'checksum': ?instance.checksum?.toJson(),
};
@@ -126,7 +126,7 @@ class TaskRecord {
IconData get icon => taskFileTypeIcon(name, isFolder: isFolder, protocol: protocol);
factory TaskRecord.fromApi(api_task.Task task) {
final totalBytes = task.meta.res?.size ?? 0;
final totalBytes = task.meta.res?.size ?? task.meta.opts.size;
final downloadedBytes = task.progress.downloaded;
final progress = totalBytes > 0 ? (downloadedBytes / totalBytes).clamp(0.0, 1.0) : null;
final status = _statusFromApi(task.status);
@@ -72,6 +72,7 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
String _httpMethod = 'GET';
String _httpBody = '';
String? _initialRawUrl;
int _initialSize = 0;
Map<String, String>? _initialLabels;
RequestProxyMode _proxyMode = RequestProxyMode.follow;
String _proxyScheme = 'http';
@@ -182,6 +183,7 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
void _handleUrlChanged() {
final urlText = _urlController.text;
if (urlText != _lastUrlText) {
_initialSize = 0;
_lastUrlText = urlText;
_renameController.clear();
if (!_programmaticUrlChange && _fileDataUri.isNotEmpty) {
@@ -200,6 +202,7 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
final req = task.req;
final opts = task.opts;
setState(() {
_initialSize = 0;
_checksumAlgorithm = 'md5';
_checksumHashController.clear();
if (req?.url.isNotEmpty == true) {
@@ -239,6 +242,7 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
}
}
if (opts != null) {
_initialSize = opts.size;
if (opts.path.isNotEmpty) {
_directoryController.text = opts.path;
}
@@ -1084,15 +1088,14 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
}
api_options.Options _buildOptions({String? name, List<int> selectFiles = const [], bool omitChecksum = false}) {
final urls = _inputUrls();
final connections = int.tryParse(_connectionsController.text.trim()) ?? 0;
final checksumExpected = _checksumHashController.text.trim();
final checksum = (!omitChecksum && _inputUrls().length <= 1 && checksumExpected.isNotEmpty)
? api_options.ChecksumOption(
algorithm: _checksumAlgorithm,
expected: checksumExpected,
)
? api_options.ChecksumOption(algorithm: _checksumAlgorithm, expected: checksumExpected)
: null;
return api_options.Options(
size: urls.length == 1 && _parseProtocol(_protocolUrlFor(urls.first)) == _TaskProtocol.http ? _initialSize : 0,
name: name ?? _renameController.text.trim(),
path: _directoryController.text.trim(),
asDefaultPath: _asDefaultPath,
@@ -1135,16 +1138,25 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
if (result.res.files.isNotEmpty) {
final reqs = selected.map((index) {
final file = result.res.files[index];
final fileRequest = file.req ?? request;
return CreateTaskBatchItem(
req: file.req ?? request,
req: fileRequest,
opts: api_options.Options(
size: _parseProtocol(fileRequest.url) == _TaskProtocol.http ? file.size : 0,
name: file.name,
path: path.join(_directoryController.text.trim(), result.res.name, file.path),
extra: _buildOptions(omitChecksum: isMultiFile).extra,
),
);
}).toList();
await ref.read(gopeedServiceProvider).createTaskBatch(CreateTaskBatch(reqs: reqs, opts: _buildOptions(omitChecksum: isMultiFile)));
await ref
.read(gopeedServiceProvider)
.createTaskBatch(
CreateTaskBatch(
reqs: reqs,
opts: _buildOptions(omitChecksum: isMultiFile),
),
);
return;
}
@@ -236,7 +236,6 @@ class _FileDownloadProgress extends StatelessWidget {
final sizeKnown = selectedFiles.every((file) => file.size > 0);
final size = sizeKnown ? selectedFiles.fold<int>(0, (total, file) => total + file.size) : 0;
final progress = size > 0 ? (downloaded / size).clamp(0.0, 1.0) : null;
final completed = progress != null && progress >= 1;
final progressLabel = progress == null ? Util.fmtByte(downloaded) : '${(progress * 100).toStringAsFixed(0)}%';
return SizedBox(
@@ -244,16 +243,14 @@ class _FileDownloadProgress extends StatelessWidget {
width: 116,
child: _TaskFileColumns(
progress: Semantics(
label: completed ? context.l10n.completed : progressLabel,
child: completed
? Icon(shad.BootstrapIcons.checkCircleFill, size: 16, color: palette.brandProgress)
: shad.CircularProgressIndicator(
value: progress ?? (downloaded > 0 ? null : 0),
size: 16,
strokeWidth: 2,
color: palette.brandProgress,
backgroundColor: palette.progressTrack,
),
label: progressLabel,
child: shad.CircularProgressIndicator(
value: progress ?? (downloaded > 0 ? null : 0),
size: 16,
strokeWidth: 2,
color: palette.brandProgress,
backgroundColor: palette.progressTrack,
),
),
size: _formatFileSize(node.size),
),
@@ -44,6 +44,30 @@ void main() {
expect(record.totalBytes, 60);
});
test('underestimated HTTP size caps progress without completing the task', () {
final task = buildTask([], 3)..protocol = api.Protocol.http;
task.progress.speed = 2;
final record = TaskRecord.fromApi(task);
expect(record.progress, 1);
expect(record.status, TaskStatus.downloading);
expect(record.downloadedBytes, 10);
expect(record.totalBytes, 3);
expect(record.remainingSeconds, isNull);
expect(const TaskFileNode(path: '', name: 'video.mp4', sizeBytes: 3, downloadedBytes: 10).progress, 1);
});
test('queued task uses the creation estimate until a resolved size is available', () {
final task = buildTask([], 0)..protocol = api.Protocol.http;
task.meta.opts.size = 1024;
task.meta.res = null;
task.status = api.Status.wait;
expect(TaskRecord.fromApi(task).totalBytes, 1024);
task.meta.res = buildTask([], 0).meta.res;
expect(TaskRecord.fromApi(task).totalBytes, isNull); // A confirmed zero must not fall back to 1024.
task.meta.res!.size = 512;
expect(TaskRecord.fromApi(task).totalBytes, 512);
});
test('completion time maps from api and stays null for legacy tasks', () {
final completed = TaskRecord.fromApi(buildTask([], 60, doneAt: '2026-01-02T03:04:05Z'));
expect(completed.doneAt, DateTime.parse('2026-01-02T03:04:05Z'));
@@ -1,3 +1,5 @@
import 'dart:convert';
import 'package:flutter/widgets.dart';
import 'package:flutter/services.dart';
import 'package:desktop_multi_window/desktop_multi_window.dart';
@@ -8,6 +10,7 @@ import 'package:flutter_riverpod/flutter_riverpod.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:go_router/go_router.dart';
import 'package:gopeed/api/model/create_task.dart';
import 'package:gopeed/api/model/create_task_batch.dart';
import 'package:gopeed/api/model/downloader_config.dart';
import 'package:gopeed/api/model/options.dart';
import 'package:gopeed/api/model/request.dart';
@@ -30,6 +33,73 @@ import 'package:gopeed/l10n/app_localizations.dart';
import 'package:shadcn_flutter/shadcn_flutter.dart' as shad;
void main() {
testWidgets('extension file estimates survive batch submission and window serialization', (tester) async {
tester.view.physicalSize = const Size(900, 1000);
tester.view.devicePixelRatio = 1;
addTearDown(tester.view.resetPhysicalSize);
addTearDown(tester.view.resetDevicePixelRatio);
CreateTaskBatch? submitted;
final registry = CapabilityRegistry(createAppCapabilityCodecs())
..bind(GopeedMethods.getConfig, (_) => DownloaderConfig(downloadDir: '/downloads'))
..bind(
GopeedMethods.resolve,
(_) => ResolveResult(
id: '',
res: Resource(
files: [
FileInfo(
name: 'under.mp4',
size: 3,
req: Request(url: 'https://example.com/video'),
),
FileInfo(
name: 'over.mp4',
size: 1048576,
req: Request(url: 'https://example.com/audio'),
),
],
),
),
)
..bind(GopeedMethods.createTaskBatch, (batch) {
submitted = CreateTaskBatch.fromJson(jsonDecode(jsonEncode(batch)) as Map<String, dynamic>);
return ['first', 'second'];
})
..bind(StorageMethods.saveCreateHistory, (_) => const RpcUnit());
final router = GoRouter(
initialLocation: '/create',
routes: [
GoRoute(path: '/', builder: (_, _) => const SizedBox()),
GoRoute(path: '/create', builder: (_, _) => const CreateTaskWindowPage()),
],
);
addTearDown(router.dispose);
await tester.pumpWidget(
ProviderScope(
overrides: [appCapabilitiesProvider.overrideWithValue(AppCapabilities(LocalCapabilityInvoker(registry)))],
child: shad.ShadcnApp.router(
theme: AppTheme.light(),
materialTheme: AppTheme.materialLight(),
routerConfig: router,
builder: (_, child) => AppComponentThemes(child: child!),
),
),
);
await tester.pumpAndSettle();
await tester.enterText(find.byKey(const ValueKey('create-task-url-input')), 'https://example.com/watch');
await tester.tap(find.byKey(const ValueKey('create-task-confirm-button')));
await tester.pump();
await tester.pump(const Duration(milliseconds: 300));
await tester.tap(find.byKey(const ValueKey('resolve-create-button')));
await tester.pumpAndSettle();
expect(submitted?.reqs?.map((item) => item.opts?.size), [3, 1048576]);
expect(submitted?.reqs?.map((item) => item.req?.url), [
'https://example.com/video',
'https://example.com/audio',
]);
expect(tester.takeException(), isNull);
});
for (final navigationFails in [false, true]) {
testWidgets('child creation selects downloads before closing (navigation fails: $navigationFails)', (tester) async {
tester.view.physicalSize = const Size(1024, 720);
@@ -264,6 +334,7 @@ void main() {
skipVerifyCert: true,
),
opts: Options(
size: 1234,
name: 'archive.zip',
path: 'D:/Downloads',
extra: OptsExtraHttp(
@@ -457,6 +528,7 @@ void main() {
expect(request?.skipVerifyCert, isTrue);
expect(ReqExtraHttp.fromJson(request?.extra! as Map<String, dynamic>).toJson(), initialTask.req?.extra);
expect(submitted?.opts?.path, 'G:/Manually selected');
expect(submitted?.opts?.size, 1234);
final options = OptsExtraHttp.fromJson(submitted?.opts?.extra! as Map<String, dynamic>);
expect(options.connections, 12);
expect(options.autoTorrent, isTrue);
@@ -676,7 +748,10 @@ void main() {
// Expand advanced
await tester.tap(find.text('Advanced'));
await tester.pumpAndSettle();
expect(_fieldText(tester, 'create-task-checksum-expected'), 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855');
expect(
_fieldText(tester, 'create-task-checksum-expected'),
'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
);
// Task B form 1: null opts
pending.set(CreateTask(req: Request(url: 'https://example.com/taskB1.zip')));
@@ -685,7 +760,12 @@ void main() {
expect(_fieldText(tester, 'create-task-checksum-expected'), '');
// Task B form 2: empty Options
pending.set(CreateTask(req: Request(url: 'https://example.com/taskB2.zip'), opts: Options()));
pending.set(
CreateTask(
req: Request(url: 'https://example.com/taskB2.zip'),
opts: Options(),
),
);
router.go('/create');
await tester.pumpAndSettle();
expect(_fieldText(tester, 'create-task-checksum-expected'), '');
@@ -804,7 +884,10 @@ void main() {
submitted = null;
await tester.tap(find.text('MD5'));
await tester.pumpAndSettle();
await tester.enterText(find.byKey(const ValueKey('create-task-checksum-expected')), '0123456789ABCDEF0123456789ABCDEF');
await tester.enterText(
find.byKey(const ValueKey('create-task-checksum-expected')),
'0123456789ABCDEF0123456789ABCDEF',
);
await tester.tap(find.text('Confirm'));
await tester.pumpAndSettle();
expect(submitted, isNotNull);
@@ -821,7 +904,10 @@ void main() {
await tester.pumpAndSettle();
await tester.tap(find.text('SHA-1'));
await tester.pumpAndSettle();
await tester.enterText(find.byKey(const ValueKey('create-task-checksum-expected')), 'da39a3ee5e6b4b0d3255bfef95601890afd80709');
await tester.enterText(
find.byKey(const ValueKey('create-task-checksum-expected')),
'da39a3ee5e6b4b0d3255bfef95601890afd80709',
);
await tester.tap(find.text('Confirm'));
await tester.pumpAndSettle();
expect(submitted, isNotNull);
@@ -838,7 +924,10 @@ void main() {
await tester.pumpAndSettle();
await tester.tap(find.text('SHA-256'));
await tester.pumpAndSettle();
await tester.enterText(find.byKey(const ValueKey('create-task-checksum-expected')), 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855');
await tester.enterText(
find.byKey(const ValueKey('create-task-checksum-expected')),
'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
);
await tester.tap(find.text('Confirm'));
await tester.pumpAndSettle();
expect(submitted, isNotNull);
@@ -0,0 +1,75 @@
import 'package:flutter/widgets.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:gopeed/api/model/task.dart' as api;
import 'package:gopeed/features/tasks/domain/task_record.dart';
import 'package:gopeed/features/tasks/presentation/widgets/task_file_tree.dart';
import 'package:gopeed/features/tasks/presentation/widgets/task_progress_bar.dart';
import 'package:gopeed/shared/theme/app_theme.dart';
import 'package:shadcn_flutter/shadcn_flutter.dart' as shad;
void main() {
for (final scenario in [
(protocol: api.Protocol.http, completed: false),
(protocol: api.Protocol.http, completed: true),
(protocol: api.Protocol.bt, completed: false),
]) {
testWidgets('file progress stays bounded and respects completion state: $scenario', (tester) async {
tester.view.physicalSize = const Size(520, 420);
tester.view.devicePixelRatio = 1;
addTearDown(tester.view.resetPhysicalSize);
addTearDown(tester.view.resetDevicePixelRatio);
final total = scenario.completed ? 10 : 3;
final task = TaskRecord(
id: 'estimated-size',
name: 'video.mp4',
protocol: scenario.protocol,
status: scenario.completed ? TaskStatus.completed : TaskStatus.downloading,
downloaded: '10 B',
total: '$total B',
progress: 1,
url: 'https://example.com/video',
storagePath: '/downloads/video.mp4',
files: [TaskFileNode(path: '', name: 'video.mp4', sizeBytes: total, resourceIndex: 0)],
uploading: false,
);
final runtime = api.TaskRuntimeStatus(
status: scenario.completed ? api.Status.done : api.Status.running,
used: 0,
speed: 2,
downloaded: 10,
total: total,
uploadSpeed: 0,
uploaded: 0,
files: [api.FileRuntimeStatus(index: 0, size: total, downloaded: 10)],
);
await tester.pumpWidget(
shad.ShadcnApp(
theme: AppTheme.light(),
materialTheme: AppTheme.materialLight(),
home: Padding(
padding: const EdgeInsets.all(24),
child: TaskFileTree(task: task, runtimeStatus: runtime),
),
),
);
await tester.pump();
final progress = tester.widget<TaskProgressBar>(find.byKey(const ValueKey('task-files-total-progress')));
expect(progress.value, 1);
expect(find.text('100%'), findsOneWidget);
expect(find.text('333%'), findsNothing);
final file = find.byKey(const ValueKey('task-file-progress-file:0'));
final check = find.descendant(of: file, matching: find.byIcon(shad.BootstrapIcons.checkCircleFill));
final ring = find.descendant(of: file, matching: find.byType(shad.CircularProgressIndicator));
if (!scenario.completed) {
expect(check, findsNothing);
expect(ring, findsOneWidget);
expect(tester.widget<shad.CircularProgressIndicator>(ring).value, 1);
} else {
expect(check, findsOneWidget);
expect(ring, findsNothing);
}
expect(tester.takeException(), isNull);
await tester.pumpWidget(const SizedBox());
});
}
}