mirror of
https://github.com/GopeedLab/gopeed.git
synced 2026-11-04 08:10:55 +00:00
feat(http): support advisory extension download sizes (#1599)
This commit is contained in:
17 files changed
+1047
-182
No files matched your search
@@ -316,13 +316,17 @@ type Fetcher struct {
|
||||
resolveFallback atomic.Bool // Resolve prefetch is still available until the first Range request takes over
|
||||
|
||||
// Target file
|
||||
file *os.File
|
||||
fileMu sync.Mutex
|
||||
redirectURL string
|
||||
redirectLock sync.Mutex
|
||||
ifRange string
|
||||
rangeReprobeEligible bool
|
||||
rangeValidatorPinned bool
|
||||
file *os.File
|
||||
fileMu sync.Mutex
|
||||
redirectURL string
|
||||
redirectLock sync.Mutex
|
||||
ifRange string
|
||||
rangeReprobeEligible bool
|
||||
rangeValidatorPinned bool
|
||||
// sizeKnown distinguishes a display estimate from an HTTP-confirmed length,
|
||||
// including zero. Nil preserves the interpretation of legacy snapshots.
|
||||
sizeKnown *bool
|
||||
// An interrupted chunked sequential response must restart from byte zero.
|
||||
sequentialSizeUnknown bool
|
||||
|
||||
// Lifecycle control
|
||||
@@ -369,6 +373,40 @@ func (f *Fetcher) setState(s fetcherState) {
|
||||
f.state.Store(int32(s))
|
||||
}
|
||||
|
||||
// Size state is read under connMu or before download workers are started.
|
||||
func (f *Fetcher) hasKnownSize() bool {
|
||||
if f.sizeKnown != nil {
|
||||
return *f.sizeKnown
|
||||
}
|
||||
return f.meta != nil && f.meta.Res != nil && f.meta.Res.Size > 0 && !f.sequentialSizeUnknown
|
||||
}
|
||||
|
||||
func (f *Fetcher) knownSize() int64 {
|
||||
if f.hasKnownSize() {
|
||||
return f.meta.Res.Size
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (f *Fetcher) sizeHint() int64 {
|
||||
if f.meta.Opts != nil && f.meta.Opts.Size > 0 {
|
||||
return f.meta.Opts.Size
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// setResourceSizeLocked publishes the value and its authority in one snapshot.
|
||||
func (f *Fetcher) setResourceSizeLocked(size int64, known bool) {
|
||||
if !known {
|
||||
size = f.sizeHint()
|
||||
}
|
||||
f.sizeKnown = &known
|
||||
f.meta.Res.Size = size
|
||||
if len(f.meta.Res.Files) > 0 && f.meta.Res.Files[0] != nil {
|
||||
f.meta.Res.Files[0].Size = size
|
||||
}
|
||||
}
|
||||
|
||||
// updateMaxConnTime updates maxConnTime if the new duration is larger
|
||||
func (f *Fetcher) updateMaxConnTime(d time.Duration) {
|
||||
newVal := int64(d)
|
||||
@@ -447,6 +485,13 @@ func (f *Fetcher) Resolve(req *base.Request) error {
|
||||
}
|
||||
}
|
||||
}
|
||||
// A hint is only a display value, even when the origin advertises Range.
|
||||
known := resp.StatusCode == base.HttpCodePartialContent || resp.ContentLength >= 0
|
||||
f.sizeKnown = &known
|
||||
f.sequentialSizeUnknown = !res.Range && !known
|
||||
if !known {
|
||||
res.Size = f.sizeHint()
|
||||
}
|
||||
|
||||
// Parse last modified time
|
||||
var lastModifiedTime *time.Time
|
||||
@@ -803,7 +848,7 @@ func (f *Fetcher) doStart() error {
|
||||
_, err = os.Stat(name)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
file, err = f.ctl.Touch(name, f.meta.Res.Size)
|
||||
file, err = f.ctl.Touch(name, f.knownSize())
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
@@ -821,7 +866,7 @@ func (f *Fetcher) doStart() error {
|
||||
// For non-range resources, the response will be used directly
|
||||
var prefetchedBytes int64
|
||||
if f.meta.Res.Range {
|
||||
resolveComplete := f.meta.Res.Size > 0 && f.prefetchDone.Load() && f.prefetchErr == nil && f.prefetchSize.Load() >= f.meta.Res.Size
|
||||
resolveComplete := f.knownSize() > 0 && f.prefetchDone.Load() && f.prefetchErr == nil && f.prefetchSize.Load() >= f.knownSize()
|
||||
if f.resolveFallback.Load() && !resolveComplete {
|
||||
// Keep Resolve downloading until the first Range response has been
|
||||
// validated. The Range request starts at a stable snapshot; a valid
|
||||
@@ -993,7 +1038,7 @@ func (f *Fetcher) expandConnections() {
|
||||
return
|
||||
}
|
||||
|
||||
totalSize := f.meta.Res.Size
|
||||
totalSize := f.knownSize()
|
||||
|
||||
f.connMu.Lock()
|
||||
|
||||
@@ -1284,7 +1329,7 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
|
||||
rangeMode := f.meta.Res.Range
|
||||
ifRange := ""
|
||||
if resumeProbe {
|
||||
end := f.meta.Res.Size - 1
|
||||
end := f.knownSize() - 1
|
||||
rangeEnd = &end
|
||||
ifRange = f.ifRange
|
||||
} else if rangeMode && (f.rangeValidatorPinned || f.resolveFallback.Load()) {
|
||||
@@ -1364,7 +1409,7 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
|
||||
}
|
||||
if rangeRequested && resp.StatusCode == base.HttpCodePartialContent {
|
||||
var responseTotal int64
|
||||
expectedResponseLength, responseTotal, err = validateRangeResponse(resp, rangeStart, rangeEnd, f.meta.Res.Size)
|
||||
expectedResponseLength, responseTotal, err = validateRangeResponse(resp, rangeStart, rangeEnd, f.knownSize())
|
||||
if err != nil {
|
||||
resp.Body.Close()
|
||||
return err
|
||||
@@ -1389,7 +1434,7 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
|
||||
f.rangeValidatorPinned = true
|
||||
f.sequentialSizeUnknown = false
|
||||
conn.Chunk.Begin = 0
|
||||
conn.Chunk.setEnd(f.meta.Res.Size - 1)
|
||||
conn.Chunk.setEnd(f.knownSize() - 1)
|
||||
f.connMu.Unlock()
|
||||
}
|
||||
}
|
||||
@@ -1414,7 +1459,7 @@ func (f *Fetcher) downloadChunkOnce(conn *connection, client *http.Client, buf [
|
||||
resp.Body.Close()
|
||||
return err
|
||||
}
|
||||
} else if f.meta.Res.Size > 0 {
|
||||
} else if f.hasKnownSize() && f.meta.Res.Size > 0 {
|
||||
expectedResponseLength = f.meta.Res.Size
|
||||
if resp.ContentLength >= 0 && resp.ContentLength != expectedResponseLength {
|
||||
resp.Body.Close()
|
||||
@@ -1550,7 +1595,7 @@ func (f *Fetcher) commitResolveTakeover(conn *connection, requestedStart int64)
|
||||
}
|
||||
|
||||
prefetched := f.stopPrefetchAndCopyData()
|
||||
if prefetched < requestedStart || prefetched > f.meta.Res.Size {
|
||||
if prefetched < requestedStart || prefetched > f.knownSize() {
|
||||
return 0, fmt.Errorf("resolve prefetch advanced to invalid offset %d for range start %d", prefetched, requestedStart)
|
||||
}
|
||||
f.resolveDataPos.Store(prefetched)
|
||||
@@ -1581,7 +1626,7 @@ func (f *Fetcher) completeFromResolveFallback(conn *connection) bool {
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
knownSize := f.meta.Res.Size > 0
|
||||
knownSize := f.hasKnownSize()
|
||||
if f.prefetchErr != nil || (knownSize && f.prefetchSize.Load() != f.meta.Res.Size) {
|
||||
f.cleanupPrefetchFile()
|
||||
return false
|
||||
@@ -1595,10 +1640,7 @@ func (f *Fetcher) completeFromResolveFallback(conn *connection) bool {
|
||||
|
||||
f.connMu.Lock()
|
||||
if !knownSize {
|
||||
f.meta.Res.Size = prefetched
|
||||
if len(f.meta.Res.Files) > 0 && f.meta.Res.Files[0] != nil {
|
||||
f.meta.Res.Files[0].Size = prefetched
|
||||
}
|
||||
f.setResourceSizeLocked(prefetched, true)
|
||||
}
|
||||
f.meta.Res.Range = false
|
||||
f.rangeReprobeEligible = false
|
||||
@@ -1680,10 +1722,7 @@ func (f *Fetcher) resolveOpenEndedRange(conn *connection, totalSize int64) error
|
||||
f.connMu.Unlock()
|
||||
return err
|
||||
}
|
||||
f.meta.Res.Size = totalSize
|
||||
if len(f.meta.Res.Files) > 0 && f.meta.Res.Files[0] != nil {
|
||||
f.meta.Res.Files[0].Size = totalSize
|
||||
}
|
||||
f.setResourceSizeLocked(totalSize, true)
|
||||
conn.Chunk.setEnd(totalSize - 1)
|
||||
f.connMu.Unlock()
|
||||
return nil
|
||||
@@ -1755,10 +1794,7 @@ func (f *Fetcher) resizeSequentialResource(size int64) error {
|
||||
return err
|
||||
}
|
||||
|
||||
f.meta.Res.Size = size
|
||||
if len(f.meta.Res.Files) > 0 {
|
||||
f.meta.Res.Files[0].Size = size
|
||||
}
|
||||
f.setResourceSizeLocked(size, true)
|
||||
f.sequentialSizeUnknown = false
|
||||
f.connMu.Unlock()
|
||||
return nil
|
||||
@@ -1783,10 +1819,7 @@ func (f *Fetcher) beginUnknownSizeSequentialResource() error {
|
||||
return err
|
||||
}
|
||||
|
||||
f.meta.Res.Size = 0
|
||||
if len(f.meta.Res.Files) > 0 {
|
||||
f.meta.Res.Files[0].Size = 0
|
||||
}
|
||||
f.setResourceSizeLocked(0, false)
|
||||
f.sequentialSizeUnknown = true
|
||||
f.ifRange = ""
|
||||
f.rangeValidatorPinned = false
|
||||
@@ -1870,11 +1903,9 @@ func (f *Fetcher) runConnectionWithResolveResp(conn *connection) {
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
f.connMu.Lock()
|
||||
if f.meta.Res.Size == 0 {
|
||||
f.meta.Res.Size = conn.Downloaded
|
||||
if len(f.meta.Res.Files) > 0 && f.meta.Res.Files[0] != nil {
|
||||
f.meta.Res.Files[0].Size = conn.Downloaded
|
||||
}
|
||||
if !f.hasKnownSize() {
|
||||
f.setResourceSizeLocked(conn.Downloaded, true)
|
||||
f.sequentialSizeUnknown = false
|
||||
}
|
||||
conn.Completed = true
|
||||
conn.State = connCompleted
|
||||
@@ -1913,78 +1944,15 @@ func (f *Fetcher) runConnectionFallback(conn *connection) {
|
||||
}
|
||||
|
||||
f.connMu.Lock()
|
||||
if retries > 0 {
|
||||
f.resetConnectionForRestart(conn)
|
||||
}
|
||||
conn.State = connConnecting
|
||||
f.connMu.Unlock()
|
||||
|
||||
err := func() error {
|
||||
httpReq, err := f.buildRequest(conn.ctx, f.meta.Req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Record connection start time for adaptive timeout tracking
|
||||
connStartTime := time.Now()
|
||||
|
||||
resp, err := client.Do(httpReq)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != base.HttpCodeOK && resp.StatusCode != base.HttpCodePartialContent {
|
||||
return NewRequestError(resp.StatusCode)
|
||||
}
|
||||
|
||||
// Record successful connection time for adaptive timeout
|
||||
f.updateMaxConnTime(time.Since(connStartTime))
|
||||
|
||||
f.connMu.Lock()
|
||||
conn.State = connDownloading
|
||||
conn.failed = false
|
||||
f.connMu.Unlock()
|
||||
|
||||
f.primaryReadyOnce.Do(func() {
|
||||
close(f.primaryReadyCh)
|
||||
})
|
||||
if f.slowStart != nil {
|
||||
f.slowStart.onConnectSuccess()
|
||||
}
|
||||
|
||||
reader := f.responseReader(resp)
|
||||
for {
|
||||
if conn.ctx.Err() != nil {
|
||||
return conn.ctx.Err()
|
||||
}
|
||||
|
||||
n, err := reader.Read(buf)
|
||||
if n > 0 {
|
||||
f.connMu.Lock()
|
||||
writeOffset := conn.Chunk.Downloaded
|
||||
f.connMu.Unlock()
|
||||
|
||||
f.fileMu.Lock()
|
||||
if f.file != nil {
|
||||
_, writeErr := f.file.WriteAt(buf[:n], writeOffset)
|
||||
if writeErr != nil {
|
||||
f.fileMu.Unlock()
|
||||
return writeErr
|
||||
}
|
||||
}
|
||||
f.fileMu.Unlock()
|
||||
|
||||
f.connMu.Lock()
|
||||
conn.Chunk.Downloaded += int64(n)
|
||||
conn.Downloaded += int64(n)
|
||||
f.connMu.Unlock()
|
||||
}
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
}()
|
||||
// Restored tasks without a reusable Resolve response need the same
|
||||
// length handling and restart semantics as other sequential requests.
|
||||
err := f.downloadChunkOnce(conn, client, buf)
|
||||
|
||||
if err == nil {
|
||||
f.connMu.Lock()
|
||||
@@ -2133,14 +2101,14 @@ func (f *Fetcher) resetConnectionForRestart(conn *connection) {
|
||||
// hasSequentialPrefixLocked reports whether a single sequential connection has
|
||||
// a useful contiguous prefix. The caller must hold connMu.
|
||||
func (f *Fetcher) hasSequentialPrefixLocked(conn *connection) bool {
|
||||
if f.meta == nil || f.meta.Res == nil || f.meta.Res.Range || !f.rangeReprobeEligible || f.meta.Res.Size <= 0 {
|
||||
if f.meta == nil || f.meta.Res == nil || f.meta.Res.Range || !f.rangeReprobeEligible || f.knownSize() <= 0 {
|
||||
return false
|
||||
}
|
||||
if conn == nil || conn.ID != 0 || len(f.connections) != 1 || conn.Chunk == nil {
|
||||
return false
|
||||
}
|
||||
downloaded := conn.Chunk.Downloaded
|
||||
return downloaded > 0 && downloaded < f.meta.Res.Size
|
||||
return downloaded > 0 && downloaded < f.knownSize()
|
||||
}
|
||||
|
||||
// canProbeSequentialResumeLocked reports whether the prefix can be resumed
|
||||
@@ -2226,7 +2194,7 @@ func (f *Fetcher) onDownloadComplete() {
|
||||
needsMoreData := false
|
||||
if f.meta.Res.Range {
|
||||
needsMoreData = conn.Chunk != nil && (conn.Chunk.openEnded() || conn.Chunk.remain() > 0)
|
||||
} else if f.meta.Res.Size > 0 {
|
||||
} else if f.hasKnownSize() && f.meta.Res.Size > 0 {
|
||||
needsMoreData = conn.Downloaded < f.meta.Res.Size
|
||||
} else {
|
||||
needsMoreData = !conn.Completed && conn.State != connCompleted
|
||||
@@ -2236,7 +2204,7 @@ func (f *Fetcher) onDownloadComplete() {
|
||||
// This connection has remaining work and isn't done
|
||||
// Check if it failed with 403 (server limit) - these can be ignored if other connections completed the work
|
||||
if conn.State == connFailed && conn.failed {
|
||||
if re := extractRequestError(conn.lastErr); re != nil && re.Code == 403 {
|
||||
if re := extractRequestError(conn.lastErr); re != nil && re.Code == 403 && f.hasKnownSize() {
|
||||
// 403 is server connection limit, check if other connections will complete this chunk
|
||||
continue
|
||||
}
|
||||
@@ -2247,7 +2215,7 @@ func (f *Fetcher) onDownloadComplete() {
|
||||
}
|
||||
|
||||
// If total downloaded matches file size, consider it a success regardless of connection failures
|
||||
downloadComplete := f.meta.Res.Size > 0 && totalDownloaded >= f.meta.Res.Size
|
||||
downloadComplete := f.hasKnownSize() && f.meta.Res.Size > 0 && totalDownloaded >= f.meta.Res.Size
|
||||
|
||||
// Integrity errors must win over byte-count completion. A malformed response
|
||||
// may be detected only after the expected bytes have already been written.
|
||||
@@ -2264,7 +2232,7 @@ func (f *Fetcher) onDownloadComplete() {
|
||||
for _, conn := range f.connections {
|
||||
if conn.State == connFailed && conn.failed {
|
||||
// Skip 403 errors (server connection limit) - these are expected when exceeding server's limit
|
||||
if re := extractRequestError(conn.lastErr); re != nil && re.Code == 403 {
|
||||
if re := extractRequestError(conn.lastErr); re != nil && re.Code == 403 && f.hasKnownSize() {
|
||||
continue
|
||||
}
|
||||
if re := extractRequestError(conn.lastErr); re != nil {
|
||||
@@ -2364,7 +2332,7 @@ func (f *Fetcher) checkCompletion() bool {
|
||||
totalDownloaded += conn.Downloaded
|
||||
}
|
||||
|
||||
if f.meta.Res.Size > 0 && totalDownloaded >= f.meta.Res.Size {
|
||||
if f.hasKnownSize() && f.meta.Res.Size > 0 && totalDownloaded >= f.meta.Res.Size {
|
||||
// Don't start a new goroutine - let the caller handle completion
|
||||
return true
|
||||
}
|
||||
@@ -2391,6 +2359,14 @@ func (f *Fetcher) checkCompletion() bool {
|
||||
|
||||
// Patch modifies the HTTP request information.
|
||||
func (f *Fetcher) Patch(req *base.Request, opts *base.Options) error {
|
||||
if opts != nil && opts.Size > 0 {
|
||||
f.connMu.Lock()
|
||||
f.meta.Opts.Size = opts.Size
|
||||
if f.meta.Res != nil && !f.hasKnownSize() {
|
||||
f.setResourceSizeLocked(0, false)
|
||||
}
|
||||
f.connMu.Unlock()
|
||||
}
|
||||
// Patch request info
|
||||
if req != nil {
|
||||
if req.URL != "" {
|
||||
@@ -2488,8 +2464,8 @@ func (f *Fetcher) Pause() error {
|
||||
conn := f.connections[0]
|
||||
if conn != nil && conn.ID == 0 && conn.Chunk != nil && !conn.Completed {
|
||||
conn.Chunk.Begin = prefetched
|
||||
if f.meta.Res.Size > 0 {
|
||||
conn.Chunk.setEnd(f.meta.Res.Size - 1)
|
||||
if f.knownSize() > 0 {
|
||||
conn.Chunk.setEnd(f.knownSize() - 1)
|
||||
} else {
|
||||
conn.Chunk.End = nil
|
||||
}
|
||||
|
||||
@@ -19,10 +19,11 @@ type fetcherData struct {
|
||||
IfRange string // Strong ETag or Last-Modified validator for safe resume
|
||||
RangeReprobeEligible bool // Origin advertised Range before falling back to sequential mode
|
||||
RangeValidatorPinned bool // Recovered Range mode must keep the same If-Range validator
|
||||
SequentialSizeUnknown bool // Interrupted chunked restart must retry from byte zero
|
||||
SequentialSizeUnknown bool // Interrupted chunked sequential streams restart from byte zero
|
||||
SizeKnown *bool // Nil for legacy snapshots; false means Size is only a display estimate
|
||||
Range *bool // Authoritative Range mode; nil for records saved by older versions
|
||||
ResourceSize *int64 // Authoritative resource size from the same snapshot as Connections
|
||||
FileSize *int64 // Authoritative single-file size; nil for older records
|
||||
ResourceSize *int64 // Display size from the same snapshot; authority is recorded by SizeKnown
|
||||
FileSize *int64 // Single-file size from the same snapshot; nil for older records
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
@@ -93,6 +94,7 @@ func (fm *FetcherManager) Store(f fetcher.Fetcher) (data any, err error) {
|
||||
rangeReprobeEligible := _f.rangeReprobeEligible
|
||||
rangeValidatorPinned := _f.rangeValidatorPinned
|
||||
sequentialSizeUnknown := _f.sequentialSizeUnknown
|
||||
sizeKnown := _f.hasKnownSize()
|
||||
var rangeMode *bool
|
||||
var resourceSize *int64
|
||||
var fileSize *int64
|
||||
@@ -115,6 +117,7 @@ func (fm *FetcherManager) Store(f fetcher.Fetcher) (data any, err error) {
|
||||
RangeReprobeEligible: rangeReprobeEligible,
|
||||
RangeValidatorPinned: rangeValidatorPinned,
|
||||
SequentialSizeUnknown: sequentialSizeUnknown,
|
||||
SizeKnown: &sizeKnown,
|
||||
Range: rangeMode,
|
||||
ResourceSize: resourceSize,
|
||||
FileSize: fileSize,
|
||||
@@ -168,6 +171,10 @@ func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v
|
||||
fetcher.rangeReprobeEligible = fd.RangeReprobeEligible
|
||||
fetcher.rangeValidatorPinned = fd.RangeValidatorPinned
|
||||
fetcher.sequentialSizeUnknown = fd.SequentialSizeUnknown
|
||||
if fd.SizeKnown != nil {
|
||||
known := *fd.SizeKnown
|
||||
fetcher.sizeKnown = &known
|
||||
}
|
||||
if fd.Range != nil && fetcher.meta.Res != nil {
|
||||
// Recovery-critical resource fields live in the same persisted snapshot
|
||||
// as Connections, making them authoritative over task metadata that may
|
||||
|
||||
@@ -1287,6 +1287,12 @@ func TestFetcher_ValidRangeTakesOverResolveResponse(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestFetcher_UnknownSizeRangeUsesOpenEndedWorker(t *testing.T) {
|
||||
for _, hint := range []int64{0, 3, 8 * 1024 * 1024} {
|
||||
t.Run(fmt.Sprint(hint), func(t *testing.T) { testFetcherUnknownSizeRange(t, hint) })
|
||||
}
|
||||
}
|
||||
|
||||
func testFetcherUnknownSizeRange(t *testing.T, hint int64) {
|
||||
const resolveETag = `"unknown-size-v1"`
|
||||
payload := make([]byte, 4*1024*1024)
|
||||
for i := range payload {
|
||||
@@ -1352,6 +1358,7 @@ func TestFetcher_UnknownSizeRangeUsesOpenEndedWorker(t *testing.T) {
|
||||
|
||||
f := buildFetcher()
|
||||
f.Meta().Opts = &base.Options{
|
||||
Size: hint,
|
||||
Path: t.TempDir(),
|
||||
Name: "unknown-size-range.data",
|
||||
Extra: &http.OptsExtra{
|
||||
@@ -1363,7 +1370,7 @@ func TestFetcher_UnknownSizeRangeUsesOpenEndedWorker(t *testing.T) {
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
if f.meta.Res.Size != 0 || !f.meta.Res.Range {
|
||||
if f.meta.Res.Size != hint || f.hasKnownSize() || !f.meta.Res.Range {
|
||||
t.Fatalf("resolved resource = %+v, want unknown size with Range support", f.meta.Res)
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
@@ -1595,6 +1602,7 @@ func TestFetcher_UnknownSizeRangeRemainsOpenEndedAfterPause(t *testing.T) {
|
||||
|
||||
f := buildFetcher()
|
||||
f.Meta().Opts = &base.Options{
|
||||
Size: 3,
|
||||
Path: t.TempDir(),
|
||||
Name: "pause-open-range.data",
|
||||
Extra: &http.OptsExtra{
|
||||
|
||||
@@ -0,0 +1,245 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
)
|
||||
|
||||
func TestFetcher_OptionsSizeHintAndPatch(t *testing.T) {
|
||||
for _, known := range []bool{false, true} {
|
||||
for _, payload := range []string{"", "0123456789"} {
|
||||
t.Run(fmt.Sprintf("known=%v/bytes=%d", known, len(payload)), func(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
if known {
|
||||
w.Header().Set("Content-Length", fmt.Sprint(len(payload)))
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
fmt.Fprint(w, payload)
|
||||
}))
|
||||
defer server.Close()
|
||||
f := buildFetcher()
|
||||
defer f.Close()
|
||||
f.Meta().Opts = &base.Options{Size: 7, Path: t.TempDir(), Name: "output.bin"}
|
||||
if err := f.Resolve(&base.Request{URL: server.URL}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := int64(7)
|
||||
if known {
|
||||
want = int64(len(payload))
|
||||
}
|
||||
if f.meta.Res.Size != want || f.hasKnownSize() != known {
|
||||
t.Fatalf("resolved size=%d known=%v", f.meta.Res.Size, f.hasKnownSize())
|
||||
}
|
||||
if err := f.Patch(nil, &base.Options{Size: 9}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !known {
|
||||
want = 9
|
||||
}
|
||||
if f.meta.Res.Size != want {
|
||||
t.Fatal("hint update overrode a real size or failed to update an estimate")
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Wait(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.meta.Res.Size != int64(len(payload)) || !f.hasKnownSize() {
|
||||
t.Fatalf("final size=%d known=%v", f.meta.Res.Size, f.hasKnownSize())
|
||||
}
|
||||
snapshot, err := new(FetcherManager).Store(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fd := snapshot.(*fetcherData)
|
||||
if fd.SizeKnown == nil || !*fd.SizeKnown {
|
||||
t.Fatal("snapshot lost confirmed size authority")
|
||||
}
|
||||
_, restore := new(FetcherManager).Restore()
|
||||
restored := restore(f.meta, fd).(*Fetcher)
|
||||
if !restored.hasKnownSize() || restored.knownSize() != int64(len(payload)) {
|
||||
t.Fatal("restored size lost authority, including known zero")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcher_SizeHintCannotCompleteForbiddenUnknownRange(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Range") != "" {
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Accept-Ranges", "bytes")
|
||||
w.(http.Flusher).Flush()
|
||||
fmt.Fprint(w, "0123456789")
|
||||
w.(http.Flusher).Flush()
|
||||
<-r.Context().Done()
|
||||
}))
|
||||
defer server.Close()
|
||||
f := buildFetcher()
|
||||
defer f.Close()
|
||||
f.Meta().Opts = &base.Options{Size: 3, Path: t.TempDir(), Name: "output.bin"}
|
||||
if err := f.Resolve(&base.Request{URL: server.URL}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Wait(); err == nil {
|
||||
t.Fatal("a failed unknown-length Range request was completed from an estimate")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcher_SizeHintPreservesHTTPIntegrity(t *testing.T) {
|
||||
for _, chunked := range []bool{false, true} {
|
||||
for _, broken := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("chunked=%v/broken=%v", chunked, broken), func(t *testing.T) {
|
||||
payload := []byte("0123456789")
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
if chunked && broken {
|
||||
conn, rw, err := w.(http.Hijacker).Hijack()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
defer conn.Close()
|
||||
fmt.Fprint(rw, "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\na\r\n0123456789\r\n")
|
||||
rw.Flush() // Omit the final chunk to simulate a connection interruption.
|
||||
return
|
||||
}
|
||||
if !chunked {
|
||||
length := len(payload)
|
||||
if broken {
|
||||
length++
|
||||
}
|
||||
w.Header().Set("Content-Length", fmt.Sprint(length))
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
w.Write(payload)
|
||||
}))
|
||||
defer server.Close()
|
||||
f := buildFetcher()
|
||||
defer f.Close()
|
||||
f.Meta().Opts = &base.Options{Size: 3, Path: t.TempDir(), Name: "output.bin"}
|
||||
if err := f.Resolve(&base.Request{URL: server.URL}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !chunked {
|
||||
length := int64(len(payload))
|
||||
if broken {
|
||||
length++
|
||||
}
|
||||
if f.meta.Res.Size != length || f.sequentialSizeUnknown {
|
||||
t.Fatal("HTTP length did not override size hint")
|
||||
}
|
||||
} else if f.meta.Res.Size != 3 || !f.sequentialSizeUnknown {
|
||||
t.Fatal("unknown HTTP length did not retain the display hint")
|
||||
}
|
||||
if err := f.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err := f.Wait()
|
||||
if broken {
|
||||
if err == nil {
|
||||
t.Fatal("truncated HTTP response succeeded after exceeding the size hint")
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.meta.Res.Size != int64(len(payload)) || f.sequentialSizeUnknown {
|
||||
t.Fatalf("completed size=%d unknown=%v", f.meta.Res.Size, f.sequentialSizeUnknown)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetcher_SizeHintSurvivesRestore(t *testing.T) {
|
||||
for _, partial := range []bool{false, true} {
|
||||
for _, fixedLength := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("partial=%v/fixedLength=%v", partial, fixedLength), func(t *testing.T) {
|
||||
payload := []byte("0123456789")
|
||||
var requests atomic.Int64
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
request := requests.Add(1)
|
||||
if got := r.Header.Get("Range"); got != "" {
|
||||
t.Errorf("estimated sizes must not be used for resume ranges: %s", got)
|
||||
}
|
||||
if fixedLength && request > 1 {
|
||||
w.Header().Set("Content-Length", fmt.Sprint(len(payload)))
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
if fixedLength && request == 2 {
|
||||
w.Write(payload[:4]) // Retry must discard this interrupted attempt.
|
||||
return
|
||||
}
|
||||
w.Write(payload)
|
||||
}))
|
||||
defer server.Close()
|
||||
f := buildFetcher()
|
||||
opts := &base.Options{Size: 3, Path: t.TempDir(), Name: "output.bin"}
|
||||
f.Meta().Opts = opts
|
||||
if err := f.Resolve(&base.Request{URL: server.URL}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fm := new(FetcherManager)
|
||||
snapshot, err := fm.Store(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := json.Marshal(snapshot)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var saved fetcherData
|
||||
if err := json.Unmarshal(data, &saved); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !saved.SequentialSizeUnknown || saved.SizeKnown == nil || *saved.SizeKnown || *saved.ResourceSize != 3 {
|
||||
t.Fatal("snapshot lost the size hint or its advisory state")
|
||||
}
|
||||
f.Close()
|
||||
// An interrupted earlier attempt left a tail longer than the new output.
|
||||
if err := os.WriteFile(filepath.Join(opts.Path, opts.Name), bytes.Repeat([]byte("old"), 100), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if partial {
|
||||
chunk := newChunk(0, 0)
|
||||
chunk.Downloaded = 7 // Already exceeds the display estimate.
|
||||
saved.Connections = []*connection{{ID: 0, Role: rolePrimary, State: connNotStarted, Chunk: chunk, Downloaded: 7}}
|
||||
}
|
||||
_, restore := fm.Restore()
|
||||
restored := restore(f.meta, &saved).(*Fetcher)
|
||||
restored.Setup(f.ctl)
|
||||
defer restored.Close()
|
||||
if err := restored.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := restored.Wait(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := os.ReadFile(restored.meta.SingleFilepath())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(body, payload) || restored.meta.Res.Size != int64(len(payload)) || restored.Progress().TotalDownloaded() != int64(len(payload)) {
|
||||
t.Fatalf("restored stream retained an estimate or stale bytes: body=%q size=%d progress=%d", body, restored.meta.Res.Size, restored.Progress().TotalDownloaded())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -108,6 +108,9 @@ type FileInfo struct {
|
||||
|
||||
// Options for download
|
||||
type Options struct {
|
||||
// Size is an optional HTTP download size estimate. HTTP metadata and clean EOF take precedence.
|
||||
// Other protocols ignore it.
|
||||
Size int64 `json:"size,omitempty"`
|
||||
// Download file name
|
||||
Name string `json:"name"`
|
||||
// Download file path
|
||||
|
||||
@@ -607,7 +607,7 @@ func (d *Downloader) createDirect(req *base.Request, opts *base.Options) (taskId
|
||||
return
|
||||
}
|
||||
fetcher.Meta().Req = req
|
||||
initOpt, err := d.initOptions(opts)
|
||||
initOpt, err := d.initOptions(opts, req.URL)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
@@ -654,7 +654,7 @@ func (d *Downloader) CreateWithOptions(rrId string, opts *base.Options) (taskId
|
||||
if opts == nil {
|
||||
opts = &base.Options{}
|
||||
}
|
||||
opts, err = d.initOptions(opts.Clone())
|
||||
opts, err = d.initOptions(opts.Clone(), fetcher.Meta().Req.URL)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -670,6 +670,13 @@ func (d *Downloader) CreateWithOptions(rrId string, opts *base.Options) (taskId
|
||||
if res := fetcher.Meta().Res; res != nil {
|
||||
res.CalcSize(fetcher.Meta().Opts.SelectFiles)
|
||||
}
|
||||
// Resolve has no creation options. Publish a submitted HTTP estimate now
|
||||
// while preserving any size already confirmed by the response.
|
||||
if opts.Size > 0 {
|
||||
if err := fetcher.Patch(nil, &base.Options{Size: opts.Size}); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
defer func() {
|
||||
d.fetcherMapLock.Lock()
|
||||
delete(d.fetcherCache, rrId)
|
||||
@@ -712,7 +719,7 @@ func (d *Downloader) Patch(id string, req *base.Request, opts *base.Options) err
|
||||
}
|
||||
|
||||
// Call the fetcher's Patch method
|
||||
if err := task.fetcher.Patch(req, opts); err != nil {
|
||||
if err := task.fetcher.Patch(req, httpSizeOptions(opts, task.Protocol)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1123,6 +1130,9 @@ func (d *Downloader) RuntimeStatus(id string) (*TaskRuntimeStatus, error) {
|
||||
result.ExtractProgress = task.Progress.ExtractProgress
|
||||
}
|
||||
if task.Meta == nil || task.Meta.Res == nil {
|
||||
if task.Meta != nil && task.Meta.Opts != nil && task.Protocol == "http" {
|
||||
result.Total = max(0, task.Meta.Opts.Size)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
@@ -1803,10 +1813,27 @@ func (d *Downloader) doCreate(f fetcher.Fetcher, opts *base.Options) (taskId str
|
||||
return
|
||||
}
|
||||
|
||||
func (d *Downloader) initOptions(opts *base.Options) (*base.Options, error) {
|
||||
// httpSizeOptions keeps estimates out of other protocols, including HLS URLs
|
||||
// served over HTTP. Clone before filtering because batch tasks may share options.
|
||||
func httpSizeOptions(opts *base.Options, protocol string) *base.Options {
|
||||
if opts != nil && opts.Size != 0 && (protocol != "http" || opts.Size < 0) {
|
||||
opts = opts.Clone()
|
||||
opts.Size = 0
|
||||
}
|
||||
return opts
|
||||
}
|
||||
|
||||
func (d *Downloader) initOptions(opts *base.Options, requestURL string) (*base.Options, error) {
|
||||
if opts == nil {
|
||||
opts = &base.Options{}
|
||||
}
|
||||
if opts.Size != 0 {
|
||||
fm, err := d.parseFm(requestURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
opts = httpSizeOptions(opts, fm.Name())
|
||||
}
|
||||
if opts.SelectFiles == nil {
|
||||
opts.SelectFiles = make([]int, 0)
|
||||
}
|
||||
|
||||
@@ -30,6 +30,21 @@ support must remain disabled. Do not declare the sum of the input sizes as the
|
||||
output size. The downloader reads the merged output as one file; FFmpeg errors
|
||||
propagate through the stream instead of completing a partial download.
|
||||
|
||||
Extension resolve results can set `files[].size` to an estimated output size,
|
||||
for example the sum of the selected video and audio input lengths. Clients forward
|
||||
this value as the optional task creation `opts.size` for HTTP downloads only
|
||||
(including Blob URLs served over HTTP). Other protocols, including HLS, ignore
|
||||
this option. The estimate is persisted
|
||||
and displayed while queued and downloading. A valid HTTP total length (including
|
||||
zero) always overrides it; otherwise a clean EOF records the actual output length.
|
||||
Estimates never determine byte ranges, file allocation, integrity checks, or task
|
||||
completion. Size authority is persisted alongside HTTP connection state for
|
||||
pause/retry and process restart.
|
||||
|
||||
Blob options are unchanged: `createObjectURL`'s `size` is an exact source length,
|
||||
not an estimate. Leave it unset for generated merge output and pass any estimate
|
||||
through the extension resolve file instead.
|
||||
|
||||
## JS inputs, including SABR
|
||||
|
||||
Stream inputs **must** be created by an `inputs({ signal })` callback. It returns
|
||||
@@ -130,7 +145,8 @@ associates its input progress with that Blob. No public progress callback or
|
||||
extra `createObjectURL` argument is required. Return the merge stream itself;
|
||||
wrapping it in another stream loses this association. Downloaded bytes count
|
||||
unique input offsets; speed counts received input bytes, including re-fetches.
|
||||
The task stays downloading with unknown total size while queued or merging;
|
||||
The task stays downloading while queued or merging, displaying the optional
|
||||
`files[].size` estimate (or an unknown total when omitted);
|
||||
once input traffic stops the displayed speed settles to zero. Completion uses
|
||||
the actual merged output size. Generated output waits are governed by producer
|
||||
cancellation/errors rather than the HTTP body idle timeout; HTTP media inputs
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
)
|
||||
|
||||
func TestExtensionResolveSizeThroughTaskCreation(t *testing.T) {
|
||||
for _, mode := range []string{"http", "blob"} {
|
||||
for _, known := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("%s/known=%v", mode, known), func(t *testing.T) {
|
||||
payload := []byte("0123456789")
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
body := payload
|
||||
if r.URL.Path == "/empty" {
|
||||
body = nil
|
||||
}
|
||||
if known {
|
||||
w.Header().Set("Content-Length", fmt.Sprint(len(body)))
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
w.Write(body)
|
||||
}))
|
||||
defer server.Close()
|
||||
d := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir(), TempDir: t.TempDir()})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Clear()
|
||||
d.cfg.MaxRunning = 0 // Inspect persisted queued tasks before HTTP resolution.
|
||||
extDir := t.TempDir()
|
||||
manifest := `{"name":"size-hint","author":"test","title":"Size hint","version":"1.0.0","scripts":[{"event":"onResolve","match":{"urls":["https://example.com/size-hint"]},"entry":"index.js"}]}`
|
||||
script := fmt.Sprintf(`gopeed.events.onResolve(async ctx=>{
|
||||
const mode=%q, origin=%q;
|
||||
const file=async(name,size,empty)=>({name,size,req:{url:mode==='http'?origin+(empty?'/empty':'/data'):
|
||||
await gopeed.runtime.blob.createObjectURL(()=>new ReadableStream({start(c){if(!empty)c.enqueue(new Uint8Array([48,49,50,51,52,53,54,55,56,57]));c.close();}}))}});
|
||||
ctx.res={files:[await file('under.bin',3,false),await file('over.bin',1048576,false),await file('empty.bin',99,true)]};
|
||||
});`, mode, server.URL)
|
||||
for name, contents := range map[string]string{"manifest.json": manifest, "index.js": script} {
|
||||
if err := os.WriteFile(filepath.Join(extDir, name), []byte(contents), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := d.InstallExtensionByFolder(extDir, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rr, err := d.Resolve(&base.Request{URL: "https://example.com/size-hint"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
output := t.TempDir()
|
||||
batch := &base.CreateTaskBatch{}
|
||||
for _, file := range rr.Res.Files {
|
||||
batch.Reqs = append(batch.Reqs, &base.CreateTaskBatchItem{Req: file.Req, Opts: &base.Options{Size: file.Size, Name: file.Name, Path: output}})
|
||||
}
|
||||
wire, err := json.Marshal(batch)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var submitted base.CreateTaskBatch
|
||||
if err := json.Unmarshal(wire, &submitted); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ids, err := d.CreateDirectBatch(&submitted)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i, id := range ids {
|
||||
task := d.GetTask(id)
|
||||
if task.Meta.Res != nil {
|
||||
t.Fatal("a size estimate must not mark the task resolved")
|
||||
}
|
||||
status, err := d.RuntimeStatus(id)
|
||||
if err != nil || status.Total != rr.Res.Files[i].Size {
|
||||
t.Fatalf("queued size lost: status=%+v err=%v", status, err)
|
||||
}
|
||||
var saved Task
|
||||
if _, err := d.storage.Get(bucketTask, id, &saved); err != nil || saved.Meta.Opts.Size != status.Total {
|
||||
t.Fatal("queued estimate was not persisted", err)
|
||||
}
|
||||
if err := d.doStart(task); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitForTaskTerminal(t, d, id, 5*time.Second)
|
||||
body, err := os.ReadFile(filepath.Join(output, rr.Res.Files[i].Name))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := payload
|
||||
if i == 2 {
|
||||
want = nil
|
||||
}
|
||||
if !bytes.Equal(body, want) {
|
||||
t.Fatalf("estimate changed output: %q", body)
|
||||
}
|
||||
status, err = d.RuntimeStatus(id)
|
||||
if err != nil || status.Total != int64(len(want)) || status.Downloaded != int64(len(want)) || task.Meta.Res.Files[0].Size != int64(len(want)) {
|
||||
t.Fatalf("final size not updated: %+v err=%v", status, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvedHTTPSizeEstimateAtCreation(t *testing.T) {
|
||||
for _, known := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("known=%v", known), func(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
if known {
|
||||
w.Header().Set("Content-Length", "10")
|
||||
}
|
||||
w.(http.Flusher).Flush()
|
||||
w.Write([]byte("0123456789"))
|
||||
}))
|
||||
defer server.Close()
|
||||
d := NewDownloader(&DownloaderConfig{StorageDir: t.TempDir(), TempDir: t.TempDir()})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Clear()
|
||||
d.cfg.MaxRunning = 0
|
||||
rr, err := d.Resolve(&base.Request{URL: server.URL})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id, err := d.CreateWithOptions(rr.ID, &base.Options{Size: 3, Path: t.TempDir(), Name: "output.bin"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := int64(3)
|
||||
if known {
|
||||
want = 10
|
||||
}
|
||||
status, err := d.RuntimeStatus(id)
|
||||
if err != nil || status.Total != want || d.GetTask(id).Meta.Res.Files[0].Size != want {
|
||||
t.Fatalf("resolve-before-create size=%+v want=%d err=%v", status, want, err)
|
||||
}
|
||||
if err := d.doStart(d.GetTask(id)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitForTaskTerminal(t, d, id, 5*time.Second)
|
||||
task := d.GetTask(id)
|
||||
if task.Status != base.DownloadStatusDone || task.Meta.Res.Size != 10 {
|
||||
t.Fatalf("estimate changed completion: status=%s size=%d", task.Status, task.Meta.Res.Size)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/GopeedLab/gopeed/pkg/base"
|
||||
)
|
||||
|
||||
func TestExtensionFileSizeIsAdvisory(t *testing.T) {
|
||||
video, err := os.ReadFile("engine/testdata/ffmpeg/video.mp4")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
audio, err := os.ReadFile("engine/testdata/ffmpeg/audio.mp4")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, mode := range []string{"plain", "merge", "wrapped-merge"} {
|
||||
var expected []byte
|
||||
for _, size := range []int64{0, 3, 1 << 20} {
|
||||
t.Run(fmt.Sprintf("%s/size=%d", mode, size), func(t *testing.T) {
|
||||
d := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir(), TempDir: t.TempDir(), RefreshInterval: 50})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Clear()
|
||||
e, _ := d.newExtensionEngine()
|
||||
defer e.Close()
|
||||
e.Runtime.Set("videoBytes", e.Runtime.NewArrayBuffer(video))
|
||||
e.Runtime.Set("audioBytes", e.Runtime.NewArrayBuffer(audio))
|
||||
e.Runtime.Set("mode", mode)
|
||||
result, err := e.RunString(`__gopeed_blob_create_object_url(()=>{
|
||||
const source=b=>new ReadableStream({start(c){c.enqueue(new Uint8Array(b));c.close();}});
|
||||
if (mode==='plain') return source([0,1,2,3,4,5,6,7,8,9]);
|
||||
const output=__gopeed_ffmpeg.merge({inputs:()=>({video:source(videoBytes),audio:source(audioBytes)})});
|
||||
if (mode==='merge') return output;
|
||||
const reader=output.getReader();
|
||||
return new ReadableStream({async pull(c){const {done,value}=await reader.read();if(done)c.close();else c.enqueue(value);},cancel(r){return reader.cancel(r);}});
|
||||
},{contentType:'video/mp4',range:false})`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
output := t.TempDir()
|
||||
id, err := d.CreateDirect(&base.Request{URL: fmt.Sprint(result)}, &base.Options{Size: size, Path: output, Name: "output.bin"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitForTaskTerminal(t, d, id, 30*time.Second)
|
||||
body, err := os.ReadFile(filepath.Join(output, "output.bin"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if size == 0 {
|
||||
expected = body
|
||||
if mode == "plain" && !bytes.Equal(body, []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) {
|
||||
t.Fatalf("unexpected stream output: %v", body)
|
||||
}
|
||||
if mode != "plain" && !bytes.Contains(body, []byte("moof")) {
|
||||
t.Fatal("invalid merged output")
|
||||
}
|
||||
} else if !bytes.Equal(body, expected) {
|
||||
t.Fatalf("size hint changed output: got %d bytes, want %d", len(body), len(expected))
|
||||
}
|
||||
task := d.GetTask(id)
|
||||
task.lock.Lock()
|
||||
defer task.lock.Unlock()
|
||||
if task.Meta.Res.Size != int64(len(body)) || task.Meta.Res.Files[0].Size != int64(len(body)) {
|
||||
t.Fatalf("final metadata must use actual size: %#v", task.Meta.Res)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtensionFileSizeYieldsToBlobContentLength(t *testing.T) {
|
||||
for _, materialized := range []bool{false, true} {
|
||||
for _, size := range []int64{3, 1 << 20} {
|
||||
t.Run(fmt.Sprintf("materialized=%v/estimate=%d", materialized, size), func(t *testing.T) {
|
||||
d := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir(), TempDir: t.TempDir()})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Clear()
|
||||
e, _ := d.newExtensionEngine()
|
||||
defer e.Close()
|
||||
e.Runtime.Set("materialized", materialized)
|
||||
result, err := e.RunString(`(()=>{
|
||||
const payload=new Uint8Array([0,1,2,3,4,5,6,7,8,9]);
|
||||
const source=materialized?new Blob([payload]):()=>new ReadableStream({start(c){c.enqueue(payload);c.close();}});
|
||||
return __gopeed_blob_create_object_url(source,{size:10});
|
||||
})()`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
output := t.TempDir()
|
||||
id, err := d.CreateDirect(&base.Request{URL: fmt.Sprint(result)}, &base.Options{Size: size, Path: output, Name: "output.bin"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitForTaskTerminal(t, d, id, 5*time.Second)
|
||||
body, err := os.ReadFile(filepath.Join(output, "output.bin"))
|
||||
if err != nil || !bytes.Equal(body, []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) {
|
||||
t.Fatalf("unexpected body=%v err=%v", body, err)
|
||||
}
|
||||
task := d.GetTask(id)
|
||||
if task.Meta.Res.Size != 10 || task.Meta.Res.Files[0].Size != 10 {
|
||||
t.Fatalf("Blob Content-Length did not override estimate: %+v", task.Meta.Res)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtensionFileSizeHintWaitsForEOF(t *testing.T) {
|
||||
for _, fail := range []bool{false, true} {
|
||||
for _, size := range []int64{3, 1 << 20} {
|
||||
t.Run(fmt.Sprintf("fail=%v/size=%d", fail, size), func(t *testing.T) {
|
||||
d := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir(), TempDir: t.TempDir(), RefreshInterval: 50})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Clear()
|
||||
e, _ := d.newExtensionEngine()
|
||||
defer e.Close()
|
||||
e.Runtime.Set("fail", fail)
|
||||
result, err := e.RunString(`__gopeed_blob_create_object_url(()=>new ReadableStream({
|
||||
start(c){c.enqueue(new Uint8Array([0,1,2,3,4,5,6,7,8,9]));globalThis.finish=()=>{if(fail)c.error(new Error('upstream failed'));else c.close();};}
|
||||
}),{})`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
output := t.TempDir()
|
||||
id, err := d.CreateDirect(&base.Request{URL: fmt.Sprint(result)}, &base.Options{Size: size, Path: output, Name: "output.bin"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitForTaskDownloadedBetween(t, d, id, 10, 10, 5*time.Second)
|
||||
task := d.GetTask(id)
|
||||
task.lock.Lock()
|
||||
if task.Meta.Res.Size != size || d.taskStatus(task) != base.DownloadStatusRunning {
|
||||
t.Errorf("hint must remain visible until EOF: size=%d status=%s", task.Meta.Res.Size, d.taskStatus(task))
|
||||
}
|
||||
task.lock.Unlock()
|
||||
stat, err := os.Stat(filepath.Join(output, "output.bin"))
|
||||
if err != nil || stat.Size() != 10 {
|
||||
t.Fatalf("hint must not preallocate or truncate output: stat=%v err=%v", stat, err)
|
||||
}
|
||||
if _, err := e.RunString(`finish()`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fail {
|
||||
waitForTaskStatus(t, d, id, base.DownloadStatusError, 5*time.Second)
|
||||
} else {
|
||||
waitForTaskTerminal(t, d, id, 5*time.Second)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtensionFileSizeHintPauseContinue(t *testing.T) {
|
||||
for _, size := range []int64{3, 1 << 20} {
|
||||
t.Run(fmt.Sprint(size), func(t *testing.T) {
|
||||
d := NewDownloader(&DownloaderConfig{Storage: NewMemStorage(), StorageDir: t.TempDir(), TempDir: t.TempDir(), RefreshInterval: 50})
|
||||
if err := d.Setup(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Clear()
|
||||
e, _ := d.newExtensionEngine()
|
||||
defer e.Close()
|
||||
result, err := e.RunString(`(()=>{
|
||||
let opens=0;
|
||||
return __gopeed_blob_create_object_url(()=>new ReadableStream({start(c){
|
||||
if (++opens===1) c.enqueue(new Uint8Array([0,1,2,3,4,5,6,7,8,9]));
|
||||
else {c.enqueue(new Uint8Array([5,6]));c.close();}
|
||||
}}),{});
|
||||
})()`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
output := t.TempDir()
|
||||
id, err := d.CreateDirect(&base.Request{URL: fmt.Sprint(result)}, &base.Options{Size: size, Path: output, Name: "output.bin"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitForTaskDownloadedBetween(t, d, id, 10, 10, 5*time.Second)
|
||||
filter := &TaskFilter{IDs: []string{id}}
|
||||
if err := d.Pause(filter); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := d.Continue(filter); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitForTaskTerminal(t, d, id, 5*time.Second)
|
||||
body, err := os.ReadFile(filepath.Join(output, "output.bin"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := d.GetTask(id)
|
||||
task.lock.Lock()
|
||||
defer task.lock.Unlock()
|
||||
if !bytes.Equal(body, []byte{5, 6}) || task.Meta.Res.Size != 2 {
|
||||
t.Fatalf("resumed stream kept an estimate or stale bytes: body=%v size=%d", body, task.Meta.Res.Size)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -4,13 +4,22 @@ part 'options.g.dart';
|
||||
|
||||
@JsonSerializable(explicitToJson: true)
|
||||
class Options {
|
||||
/// HTTP-only size estimate, superseded by HTTP metadata or completion.
|
||||
int size;
|
||||
String name;
|
||||
String path;
|
||||
bool asDefaultPath;
|
||||
List<int> selectFiles;
|
||||
Object? extra;
|
||||
|
||||
Options({this.name = '', this.path = '', this.asDefaultPath = false, this.selectFiles = const [], this.extra});
|
||||
Options({
|
||||
this.size = 0,
|
||||
this.name = '',
|
||||
this.path = '',
|
||||
this.asDefaultPath = false,
|
||||
this.selectFiles = const [],
|
||||
this.extra,
|
||||
});
|
||||
|
||||
factory Options.fromJson(Map<String, dynamic> json) => _$OptionsFromJson(json);
|
||||
|
||||
@@ -22,17 +31,14 @@ class ChecksumOption {
|
||||
String algorithm;
|
||||
String expected;
|
||||
|
||||
ChecksumOption({
|
||||
this.algorithm = '',
|
||||
this.expected = '',
|
||||
});
|
||||
ChecksumOption({this.algorithm = '', this.expected = ''});
|
||||
|
||||
factory ChecksumOption.fromJson(Map<String, dynamic> json) => _$ChecksumOptionFromJson(json);
|
||||
|
||||
Map<String, dynamic> toJson() => _$ChecksumOptionToJson(this);
|
||||
}
|
||||
|
||||
@JsonSerializable()
|
||||
@JsonSerializable(explicitToJson: true)
|
||||
class OptsExtraHttp {
|
||||
int connections;
|
||||
bool? autoTorrent;
|
||||
|
||||
@@ -7,14 +7,20 @@ part of 'options.dart';
|
||||
// **************************************************************************
|
||||
|
||||
Options _$OptionsFromJson(Map<String, dynamic> json) => Options(
|
||||
size: (json['size'] as num?)?.toInt() ?? 0,
|
||||
name: json['name'] as String? ?? '',
|
||||
path: json['path'] as String? ?? '',
|
||||
asDefaultPath: json['asDefaultPath'] as bool? ?? false,
|
||||
selectFiles: (json['selectFiles'] as List<dynamic>?)?.map((e) => (e as num).toInt()).toList() ?? const [],
|
||||
selectFiles:
|
||||
(json['selectFiles'] as List<dynamic>?)
|
||||
?.map((e) => (e as num).toInt())
|
||||
.toList() ??
|
||||
const [],
|
||||
extra: json['extra'],
|
||||
);
|
||||
|
||||
Map<String, dynamic> _$OptionsToJson(Options instance) => <String, dynamic>{
|
||||
'size': instance.size,
|
||||
'name': instance.name,
|
||||
'path': instance.path,
|
||||
'asDefaultPath': instance.asDefaultPath,
|
||||
@@ -22,34 +28,38 @@ Map<String, dynamic> _$OptionsToJson(Options instance) => <String, dynamic>{
|
||||
'extra': ?instance.extra,
|
||||
};
|
||||
|
||||
ChecksumOption _$ChecksumOptionFromJson(Map<String, dynamic> json) => ChecksumOption(
|
||||
algorithm: json['algorithm'] as String? ?? '',
|
||||
expected: json['expected'] as String? ?? '',
|
||||
);
|
||||
ChecksumOption _$ChecksumOptionFromJson(Map<String, dynamic> json) =>
|
||||
ChecksumOption(
|
||||
algorithm: json['algorithm'] as String? ?? '',
|
||||
expected: json['expected'] as String? ?? '',
|
||||
);
|
||||
|
||||
Map<String, dynamic> _$ChecksumOptionToJson(ChecksumOption instance) => <String, dynamic>{
|
||||
'algorithm': instance.algorithm,
|
||||
'expected': instance.expected,
|
||||
};
|
||||
Map<String, dynamic> _$ChecksumOptionToJson(ChecksumOption instance) =>
|
||||
<String, dynamic>{
|
||||
'algorithm': instance.algorithm,
|
||||
'expected': instance.expected,
|
||||
};
|
||||
|
||||
OptsExtraHttp _$OptsExtraHttpFromJson(Map<String, dynamic> json) => OptsExtraHttp(
|
||||
connections: (json['connections'] as num?)?.toInt() ?? 0,
|
||||
autoTorrent: json['autoTorrent'] as bool?,
|
||||
deleteTorrentAfterDownload: json['deleteTorrentAfterDownload'] as bool?,
|
||||
autoExtract: json['autoExtract'] as bool?,
|
||||
archivePassword: json['archivePassword'] as String? ?? '',
|
||||
deleteAfterExtract: json['deleteAfterExtract'] as bool? ?? false,
|
||||
checksum: json['checksum'] == null
|
||||
? null
|
||||
: ChecksumOption.fromJson(json['checksum'] as Map<String, dynamic>),
|
||||
);
|
||||
OptsExtraHttp _$OptsExtraHttpFromJson(Map<String, dynamic> json) =>
|
||||
OptsExtraHttp(
|
||||
connections: (json['connections'] as num?)?.toInt() ?? 0,
|
||||
autoTorrent: json['autoTorrent'] as bool?,
|
||||
deleteTorrentAfterDownload: json['deleteTorrentAfterDownload'] as bool?,
|
||||
autoExtract: json['autoExtract'] as bool?,
|
||||
archivePassword: json['archivePassword'] as String? ?? '',
|
||||
deleteAfterExtract: json['deleteAfterExtract'] as bool? ?? false,
|
||||
checksum: json['checksum'] == null
|
||||
? null
|
||||
: ChecksumOption.fromJson(json['checksum'] as Map<String, dynamic>),
|
||||
);
|
||||
|
||||
Map<String, dynamic> _$OptsExtraHttpToJson(OptsExtraHttp instance) => <String, dynamic>{
|
||||
'connections': instance.connections,
|
||||
'autoTorrent': ?instance.autoTorrent,
|
||||
'deleteTorrentAfterDownload': ?instance.deleteTorrentAfterDownload,
|
||||
'autoExtract': ?instance.autoExtract,
|
||||
'archivePassword': instance.archivePassword,
|
||||
'deleteAfterExtract': instance.deleteAfterExtract,
|
||||
'checksum': ?instance.checksum?.toJson(),
|
||||
};
|
||||
Map<String, dynamic> _$OptsExtraHttpToJson(OptsExtraHttp instance) =>
|
||||
<String, dynamic>{
|
||||
'connections': instance.connections,
|
||||
'autoTorrent': ?instance.autoTorrent,
|
||||
'deleteTorrentAfterDownload': ?instance.deleteTorrentAfterDownload,
|
||||
'autoExtract': ?instance.autoExtract,
|
||||
'archivePassword': instance.archivePassword,
|
||||
'deleteAfterExtract': instance.deleteAfterExtract,
|
||||
'checksum': ?instance.checksum?.toJson(),
|
||||
};
|
||||
@@ -126,7 +126,7 @@ class TaskRecord {
|
||||
IconData get icon => taskFileTypeIcon(name, isFolder: isFolder, protocol: protocol);
|
||||
|
||||
factory TaskRecord.fromApi(api_task.Task task) {
|
||||
final totalBytes = task.meta.res?.size ?? 0;
|
||||
final totalBytes = task.meta.res?.size ?? task.meta.opts.size;
|
||||
final downloadedBytes = task.progress.downloaded;
|
||||
final progress = totalBytes > 0 ? (downloadedBytes / totalBytes).clamp(0.0, 1.0) : null;
|
||||
final status = _statusFromApi(task.status);
|
||||
|
||||
@@ -72,6 +72,7 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
|
||||
String _httpMethod = 'GET';
|
||||
String _httpBody = '';
|
||||
String? _initialRawUrl;
|
||||
int _initialSize = 0;
|
||||
Map<String, String>? _initialLabels;
|
||||
RequestProxyMode _proxyMode = RequestProxyMode.follow;
|
||||
String _proxyScheme = 'http';
|
||||
@@ -182,6 +183,7 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
|
||||
void _handleUrlChanged() {
|
||||
final urlText = _urlController.text;
|
||||
if (urlText != _lastUrlText) {
|
||||
_initialSize = 0;
|
||||
_lastUrlText = urlText;
|
||||
_renameController.clear();
|
||||
if (!_programmaticUrlChange && _fileDataUri.isNotEmpty) {
|
||||
@@ -200,6 +202,7 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
|
||||
final req = task.req;
|
||||
final opts = task.opts;
|
||||
setState(() {
|
||||
_initialSize = 0;
|
||||
_checksumAlgorithm = 'md5';
|
||||
_checksumHashController.clear();
|
||||
if (req?.url.isNotEmpty == true) {
|
||||
@@ -239,6 +242,7 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
|
||||
}
|
||||
}
|
||||
if (opts != null) {
|
||||
_initialSize = opts.size;
|
||||
if (opts.path.isNotEmpty) {
|
||||
_directoryController.text = opts.path;
|
||||
}
|
||||
@@ -1084,15 +1088,14 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
|
||||
}
|
||||
|
||||
api_options.Options _buildOptions({String? name, List<int> selectFiles = const [], bool omitChecksum = false}) {
|
||||
final urls = _inputUrls();
|
||||
final connections = int.tryParse(_connectionsController.text.trim()) ?? 0;
|
||||
final checksumExpected = _checksumHashController.text.trim();
|
||||
final checksum = (!omitChecksum && _inputUrls().length <= 1 && checksumExpected.isNotEmpty)
|
||||
? api_options.ChecksumOption(
|
||||
algorithm: _checksumAlgorithm,
|
||||
expected: checksumExpected,
|
||||
)
|
||||
? api_options.ChecksumOption(algorithm: _checksumAlgorithm, expected: checksumExpected)
|
||||
: null;
|
||||
return api_options.Options(
|
||||
size: urls.length == 1 && _parseProtocol(_protocolUrlFor(urls.first)) == _TaskProtocol.http ? _initialSize : 0,
|
||||
name: name ?? _renameController.text.trim(),
|
||||
path: _directoryController.text.trim(),
|
||||
asDefaultPath: _asDefaultPath,
|
||||
@@ -1135,16 +1138,25 @@ class _CreateTaskWindowPageState extends ConsumerState<CreateTaskWindowPage> {
|
||||
if (result.res.files.isNotEmpty) {
|
||||
final reqs = selected.map((index) {
|
||||
final file = result.res.files[index];
|
||||
final fileRequest = file.req ?? request;
|
||||
return CreateTaskBatchItem(
|
||||
req: file.req ?? request,
|
||||
req: fileRequest,
|
||||
opts: api_options.Options(
|
||||
size: _parseProtocol(fileRequest.url) == _TaskProtocol.http ? file.size : 0,
|
||||
name: file.name,
|
||||
path: path.join(_directoryController.text.trim(), result.res.name, file.path),
|
||||
extra: _buildOptions(omitChecksum: isMultiFile).extra,
|
||||
),
|
||||
);
|
||||
}).toList();
|
||||
await ref.read(gopeedServiceProvider).createTaskBatch(CreateTaskBatch(reqs: reqs, opts: _buildOptions(omitChecksum: isMultiFile)));
|
||||
await ref
|
||||
.read(gopeedServiceProvider)
|
||||
.createTaskBatch(
|
||||
CreateTaskBatch(
|
||||
reqs: reqs,
|
||||
opts: _buildOptions(omitChecksum: isMultiFile),
|
||||
),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
@@ -236,7 +236,6 @@ class _FileDownloadProgress extends StatelessWidget {
|
||||
final sizeKnown = selectedFiles.every((file) => file.size > 0);
|
||||
final size = sizeKnown ? selectedFiles.fold<int>(0, (total, file) => total + file.size) : 0;
|
||||
final progress = size > 0 ? (downloaded / size).clamp(0.0, 1.0) : null;
|
||||
final completed = progress != null && progress >= 1;
|
||||
final progressLabel = progress == null ? Util.fmtByte(downloaded) : '${(progress * 100).toStringAsFixed(0)}%';
|
||||
|
||||
return SizedBox(
|
||||
@@ -244,16 +243,14 @@ class _FileDownloadProgress extends StatelessWidget {
|
||||
width: 116,
|
||||
child: _TaskFileColumns(
|
||||
progress: Semantics(
|
||||
label: completed ? context.l10n.completed : progressLabel,
|
||||
child: completed
|
||||
? Icon(shad.BootstrapIcons.checkCircleFill, size: 16, color: palette.brandProgress)
|
||||
: shad.CircularProgressIndicator(
|
||||
value: progress ?? (downloaded > 0 ? null : 0),
|
||||
size: 16,
|
||||
strokeWidth: 2,
|
||||
color: palette.brandProgress,
|
||||
backgroundColor: palette.progressTrack,
|
||||
),
|
||||
label: progressLabel,
|
||||
child: shad.CircularProgressIndicator(
|
||||
value: progress ?? (downloaded > 0 ? null : 0),
|
||||
size: 16,
|
||||
strokeWidth: 2,
|
||||
color: palette.brandProgress,
|
||||
backgroundColor: palette.progressTrack,
|
||||
),
|
||||
),
|
||||
size: _formatFileSize(node.size),
|
||||
),
|
||||
|
||||
@@ -44,6 +44,30 @@ void main() {
|
||||
expect(record.totalBytes, 60);
|
||||
});
|
||||
|
||||
test('underestimated HTTP size caps progress without completing the task', () {
|
||||
final task = buildTask([], 3)..protocol = api.Protocol.http;
|
||||
task.progress.speed = 2;
|
||||
final record = TaskRecord.fromApi(task);
|
||||
expect(record.progress, 1);
|
||||
expect(record.status, TaskStatus.downloading);
|
||||
expect(record.downloadedBytes, 10);
|
||||
expect(record.totalBytes, 3);
|
||||
expect(record.remainingSeconds, isNull);
|
||||
expect(const TaskFileNode(path: '', name: 'video.mp4', sizeBytes: 3, downloadedBytes: 10).progress, 1);
|
||||
});
|
||||
|
||||
test('queued task uses the creation estimate until a resolved size is available', () {
|
||||
final task = buildTask([], 0)..protocol = api.Protocol.http;
|
||||
task.meta.opts.size = 1024;
|
||||
task.meta.res = null;
|
||||
task.status = api.Status.wait;
|
||||
expect(TaskRecord.fromApi(task).totalBytes, 1024);
|
||||
task.meta.res = buildTask([], 0).meta.res;
|
||||
expect(TaskRecord.fromApi(task).totalBytes, isNull); // A confirmed zero must not fall back to 1024.
|
||||
task.meta.res!.size = 512;
|
||||
expect(TaskRecord.fromApi(task).totalBytes, 512);
|
||||
});
|
||||
|
||||
test('completion time maps from api and stays null for legacy tasks', () {
|
||||
final completed = TaskRecord.fromApi(buildTask([], 60, doneAt: '2026-01-02T03:04:05Z'));
|
||||
expect(completed.doneAt, DateTime.parse('2026-01-02T03:04:05Z'));
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import 'dart:convert';
|
||||
|
||||
import 'package:flutter/widgets.dart';
|
||||
import 'package:flutter/services.dart';
|
||||
import 'package:desktop_multi_window/desktop_multi_window.dart';
|
||||
@@ -8,6 +10,7 @@ import 'package:flutter_riverpod/flutter_riverpod.dart';
|
||||
import 'package:flutter_test/flutter_test.dart';
|
||||
import 'package:go_router/go_router.dart';
|
||||
import 'package:gopeed/api/model/create_task.dart';
|
||||
import 'package:gopeed/api/model/create_task_batch.dart';
|
||||
import 'package:gopeed/api/model/downloader_config.dart';
|
||||
import 'package:gopeed/api/model/options.dart';
|
||||
import 'package:gopeed/api/model/request.dart';
|
||||
@@ -30,6 +33,73 @@ import 'package:gopeed/l10n/app_localizations.dart';
|
||||
import 'package:shadcn_flutter/shadcn_flutter.dart' as shad;
|
||||
|
||||
void main() {
|
||||
testWidgets('extension file estimates survive batch submission and window serialization', (tester) async {
|
||||
tester.view.physicalSize = const Size(900, 1000);
|
||||
tester.view.devicePixelRatio = 1;
|
||||
addTearDown(tester.view.resetPhysicalSize);
|
||||
addTearDown(tester.view.resetDevicePixelRatio);
|
||||
CreateTaskBatch? submitted;
|
||||
final registry = CapabilityRegistry(createAppCapabilityCodecs())
|
||||
..bind(GopeedMethods.getConfig, (_) => DownloaderConfig(downloadDir: '/downloads'))
|
||||
..bind(
|
||||
GopeedMethods.resolve,
|
||||
(_) => ResolveResult(
|
||||
id: '',
|
||||
res: Resource(
|
||||
files: [
|
||||
FileInfo(
|
||||
name: 'under.mp4',
|
||||
size: 3,
|
||||
req: Request(url: 'https://example.com/video'),
|
||||
),
|
||||
FileInfo(
|
||||
name: 'over.mp4',
|
||||
size: 1048576,
|
||||
req: Request(url: 'https://example.com/audio'),
|
||||
),
|
||||
],
|
||||
),
|
||||
),
|
||||
)
|
||||
..bind(GopeedMethods.createTaskBatch, (batch) {
|
||||
submitted = CreateTaskBatch.fromJson(jsonDecode(jsonEncode(batch)) as Map<String, dynamic>);
|
||||
return ['first', 'second'];
|
||||
})
|
||||
..bind(StorageMethods.saveCreateHistory, (_) => const RpcUnit());
|
||||
final router = GoRouter(
|
||||
initialLocation: '/create',
|
||||
routes: [
|
||||
GoRoute(path: '/', builder: (_, _) => const SizedBox()),
|
||||
GoRoute(path: '/create', builder: (_, _) => const CreateTaskWindowPage()),
|
||||
],
|
||||
);
|
||||
addTearDown(router.dispose);
|
||||
await tester.pumpWidget(
|
||||
ProviderScope(
|
||||
overrides: [appCapabilitiesProvider.overrideWithValue(AppCapabilities(LocalCapabilityInvoker(registry)))],
|
||||
child: shad.ShadcnApp.router(
|
||||
theme: AppTheme.light(),
|
||||
materialTheme: AppTheme.materialLight(),
|
||||
routerConfig: router,
|
||||
builder: (_, child) => AppComponentThemes(child: child!),
|
||||
),
|
||||
),
|
||||
);
|
||||
await tester.pumpAndSettle();
|
||||
await tester.enterText(find.byKey(const ValueKey('create-task-url-input')), 'https://example.com/watch');
|
||||
await tester.tap(find.byKey(const ValueKey('create-task-confirm-button')));
|
||||
await tester.pump();
|
||||
await tester.pump(const Duration(milliseconds: 300));
|
||||
await tester.tap(find.byKey(const ValueKey('resolve-create-button')));
|
||||
await tester.pumpAndSettle();
|
||||
expect(submitted?.reqs?.map((item) => item.opts?.size), [3, 1048576]);
|
||||
expect(submitted?.reqs?.map((item) => item.req?.url), [
|
||||
'https://example.com/video',
|
||||
'https://example.com/audio',
|
||||
]);
|
||||
expect(tester.takeException(), isNull);
|
||||
});
|
||||
|
||||
for (final navigationFails in [false, true]) {
|
||||
testWidgets('child creation selects downloads before closing (navigation fails: $navigationFails)', (tester) async {
|
||||
tester.view.physicalSize = const Size(1024, 720);
|
||||
@@ -264,6 +334,7 @@ void main() {
|
||||
skipVerifyCert: true,
|
||||
),
|
||||
opts: Options(
|
||||
size: 1234,
|
||||
name: 'archive.zip',
|
||||
path: 'D:/Downloads',
|
||||
extra: OptsExtraHttp(
|
||||
@@ -457,6 +528,7 @@ void main() {
|
||||
expect(request?.skipVerifyCert, isTrue);
|
||||
expect(ReqExtraHttp.fromJson(request?.extra! as Map<String, dynamic>).toJson(), initialTask.req?.extra);
|
||||
expect(submitted?.opts?.path, 'G:/Manually selected');
|
||||
expect(submitted?.opts?.size, 1234);
|
||||
final options = OptsExtraHttp.fromJson(submitted?.opts?.extra! as Map<String, dynamic>);
|
||||
expect(options.connections, 12);
|
||||
expect(options.autoTorrent, isTrue);
|
||||
@@ -676,7 +748,10 @@ void main() {
|
||||
// Expand advanced
|
||||
await tester.tap(find.text('Advanced'));
|
||||
await tester.pumpAndSettle();
|
||||
expect(_fieldText(tester, 'create-task-checksum-expected'), 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855');
|
||||
expect(
|
||||
_fieldText(tester, 'create-task-checksum-expected'),
|
||||
'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
|
||||
);
|
||||
|
||||
// Task B form 1: null opts
|
||||
pending.set(CreateTask(req: Request(url: 'https://example.com/taskB1.zip')));
|
||||
@@ -685,7 +760,12 @@ void main() {
|
||||
expect(_fieldText(tester, 'create-task-checksum-expected'), '');
|
||||
|
||||
// Task B form 2: empty Options
|
||||
pending.set(CreateTask(req: Request(url: 'https://example.com/taskB2.zip'), opts: Options()));
|
||||
pending.set(
|
||||
CreateTask(
|
||||
req: Request(url: 'https://example.com/taskB2.zip'),
|
||||
opts: Options(),
|
||||
),
|
||||
);
|
||||
router.go('/create');
|
||||
await tester.pumpAndSettle();
|
||||
expect(_fieldText(tester, 'create-task-checksum-expected'), '');
|
||||
@@ -804,7 +884,10 @@ void main() {
|
||||
submitted = null;
|
||||
await tester.tap(find.text('MD5'));
|
||||
await tester.pumpAndSettle();
|
||||
await tester.enterText(find.byKey(const ValueKey('create-task-checksum-expected')), '0123456789ABCDEF0123456789ABCDEF');
|
||||
await tester.enterText(
|
||||
find.byKey(const ValueKey('create-task-checksum-expected')),
|
||||
'0123456789ABCDEF0123456789ABCDEF',
|
||||
);
|
||||
await tester.tap(find.text('Confirm'));
|
||||
await tester.pumpAndSettle();
|
||||
expect(submitted, isNotNull);
|
||||
@@ -821,7 +904,10 @@ void main() {
|
||||
await tester.pumpAndSettle();
|
||||
await tester.tap(find.text('SHA-1'));
|
||||
await tester.pumpAndSettle();
|
||||
await tester.enterText(find.byKey(const ValueKey('create-task-checksum-expected')), 'da39a3ee5e6b4b0d3255bfef95601890afd80709');
|
||||
await tester.enterText(
|
||||
find.byKey(const ValueKey('create-task-checksum-expected')),
|
||||
'da39a3ee5e6b4b0d3255bfef95601890afd80709',
|
||||
);
|
||||
await tester.tap(find.text('Confirm'));
|
||||
await tester.pumpAndSettle();
|
||||
expect(submitted, isNotNull);
|
||||
@@ -838,7 +924,10 @@ void main() {
|
||||
await tester.pumpAndSettle();
|
||||
await tester.tap(find.text('SHA-256'));
|
||||
await tester.pumpAndSettle();
|
||||
await tester.enterText(find.byKey(const ValueKey('create-task-checksum-expected')), 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855');
|
||||
await tester.enterText(
|
||||
find.byKey(const ValueKey('create-task-checksum-expected')),
|
||||
'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
|
||||
);
|
||||
await tester.tap(find.text('Confirm'));
|
||||
await tester.pumpAndSettle();
|
||||
expect(submitted, isNotNull);
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
import 'package:flutter/widgets.dart';
|
||||
import 'package:flutter_test/flutter_test.dart';
|
||||
import 'package:gopeed/api/model/task.dart' as api;
|
||||
import 'package:gopeed/features/tasks/domain/task_record.dart';
|
||||
import 'package:gopeed/features/tasks/presentation/widgets/task_file_tree.dart';
|
||||
import 'package:gopeed/features/tasks/presentation/widgets/task_progress_bar.dart';
|
||||
import 'package:gopeed/shared/theme/app_theme.dart';
|
||||
import 'package:shadcn_flutter/shadcn_flutter.dart' as shad;
|
||||
|
||||
void main() {
|
||||
for (final scenario in [
|
||||
(protocol: api.Protocol.http, completed: false),
|
||||
(protocol: api.Protocol.http, completed: true),
|
||||
(protocol: api.Protocol.bt, completed: false),
|
||||
]) {
|
||||
testWidgets('file progress stays bounded and respects completion state: $scenario', (tester) async {
|
||||
tester.view.physicalSize = const Size(520, 420);
|
||||
tester.view.devicePixelRatio = 1;
|
||||
addTearDown(tester.view.resetPhysicalSize);
|
||||
addTearDown(tester.view.resetDevicePixelRatio);
|
||||
final total = scenario.completed ? 10 : 3;
|
||||
final task = TaskRecord(
|
||||
id: 'estimated-size',
|
||||
name: 'video.mp4',
|
||||
protocol: scenario.protocol,
|
||||
status: scenario.completed ? TaskStatus.completed : TaskStatus.downloading,
|
||||
downloaded: '10 B',
|
||||
total: '$total B',
|
||||
progress: 1,
|
||||
url: 'https://example.com/video',
|
||||
storagePath: '/downloads/video.mp4',
|
||||
files: [TaskFileNode(path: '', name: 'video.mp4', sizeBytes: total, resourceIndex: 0)],
|
||||
uploading: false,
|
||||
);
|
||||
final runtime = api.TaskRuntimeStatus(
|
||||
status: scenario.completed ? api.Status.done : api.Status.running,
|
||||
used: 0,
|
||||
speed: 2,
|
||||
downloaded: 10,
|
||||
total: total,
|
||||
uploadSpeed: 0,
|
||||
uploaded: 0,
|
||||
files: [api.FileRuntimeStatus(index: 0, size: total, downloaded: 10)],
|
||||
);
|
||||
await tester.pumpWidget(
|
||||
shad.ShadcnApp(
|
||||
theme: AppTheme.light(),
|
||||
materialTheme: AppTheme.materialLight(),
|
||||
home: Padding(
|
||||
padding: const EdgeInsets.all(24),
|
||||
child: TaskFileTree(task: task, runtimeStatus: runtime),
|
||||
),
|
||||
),
|
||||
);
|
||||
await tester.pump();
|
||||
final progress = tester.widget<TaskProgressBar>(find.byKey(const ValueKey('task-files-total-progress')));
|
||||
expect(progress.value, 1);
|
||||
expect(find.text('100%'), findsOneWidget);
|
||||
expect(find.text('333%'), findsNothing);
|
||||
final file = find.byKey(const ValueKey('task-file-progress-file:0'));
|
||||
final check = find.descendant(of: file, matching: find.byIcon(shad.BootstrapIcons.checkCircleFill));
|
||||
final ring = find.descendant(of: file, matching: find.byType(shad.CircularProgressIndicator));
|
||||
if (!scenario.completed) {
|
||||
expect(check, findsNothing);
|
||||
expect(ring, findsOneWidget);
|
||||
expect(tester.widget<shad.CircularProgressIndicator>(ring).value, 1);
|
||||
} else {
|
||||
expect(check, findsOneWidget);
|
||||
expect(ring, findsNothing);
|
||||
}
|
||||
expect(tester.takeException(), isNull);
|
||||
await tester.pumpWidget(const SizedBox());
|
||||
});
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user