From cfdf87c3b8aad398290fac0f8989c1e4f8e4b60d Mon Sep 17 00:00:00 2001 From: Levi Date: Thu, 20 Aug 2026 20:21:02 +0800 Subject: [PATCH] feat: HTTP add TLS fingerprint fallback (#1444) --- internal/httpclient/client.go | 346 +++++++ internal/httpclient/client_test.go | 962 ++++++++++++++++++ .../httpclienttest/fingerprint_server.go | 100 ++ internal/protocol/http/fetcher.go | 12 +- internal/protocol/http/fetcher_test.go | 76 ++ internal/protocol/http/helper.go | 36 +- pkg/download/engine/engine.go | 41 +- pkg/download/engine/fetch_integration_test.go | 168 +++ pkg/download/engine/inject/fetch/fetch.js | 898 ++++++++++++++++ pkg/download/engine/inject/fetch/module.go | 619 +++++++++++ pkg/download/engine/inject/formdata/module.go | 68 -- pkg/download/engine/inject/stream/module.go | 513 ---------- .../inject/stream/module_integration_test.go | 9 +- .../engine/inject/stream/module_test.go | 191 ---- pkg/download/engine/inject/stream/stream.js | 334 ++---- pkg/download/engine/inject/xhr/module.go | 479 +-------- .../engine/inject/xhr/tls_fingerprint.go | 24 - pkg/download/engine/inject/xhr/xhr.js | 302 ++++++ pkg/download/engine/polyfill/out/index.js | 2 +- .../engine/polyfill/package-lock.json | 9 +- pkg/download/engine/polyfill/package.json | 3 +- .../patches/whatwg-fetch+3.6.20.patch | 46 - .../engine/polyfill/src/fetch/index.js | 1 - pkg/download/engine/polyfill/src/index.js | 1 - pkg/download/engine/testdata/wpt/LICENSE.md | 11 + pkg/download/engine/testdata/wpt/UPSTREAM.md | 28 + .../cors/resources/not-cors-safelisted.json | 13 + .../fetch/api/headers/header-setcookie.any.js | 266 +++++ .../fetch/api/headers/headers-basic.any.js | 275 +++++ .../fetch/api/headers/headers-casing.any.js | 54 + .../fetch/api/headers/headers-combine.any.js | 66 ++ .../fetch/api/headers/headers-errors.any.js | 96 ++ .../headers/headers-forbidden-override.any.js | 51 + .../fetch/api/headers/headers-no-cors.any.js | 59 ++ .../api/headers/headers-normalize.any.js | 56 + .../fetch/api/headers/headers-record.any.js | 357 +++++++ .../api/headers/headers-structure.any.js | 20 + .../fetch/api/request/forbidden-method.any.js | 13 + .../request-clone-readable-stream-body.any.js | 22 + ...uest-constructor-init-body-override.any.js | 37 + .../api/request/request-consume-empty.any.js | 89 ++ .../fetch/api/request/request-consume.any.js | 148 +++ .../api/request/request-disturbed.any.js | 109 ++ .../fetch/api/request/request-error.any.js | 56 + .../wpt/fetch/api/request/request-error.js | 57 ++ .../fetch/api/request/request-headers.any.js | 177 ++++ .../fetch/api/request/request-init-002.any.js | 60 ++ .../request/request-init-contenttype.any.js | 141 +++ .../api/request/request-init-priority.any.js | 26 + .../api/request/request-init-stream.any.js | 146 +++ .../api/request/request-keepalive.any.js | 17 + .../api/request/request-structure.any.js | 143 +++ .../testdata/wpt/fetch/api/resources/utils.js | 120 +++ .../response/response-consume-empty.any.js | 88 ++ .../response/response-consume-stream.any.js | 80 ++ .../fetch/api/response/response-error.any.js | 27 + .../api/response/response-from-stream.any.js | 23 + .../response/response-headers-guard.any.js | 8 + .../api/response/response-init-001.any.js | 64 ++ .../api/response/response-init-002.any.js | 61 ++ .../response/response-init-contenttype.any.js | 125 +++ .../api/response/response-static-error.any.js | 22 + .../api/response/response-static-json.any.js | 96 ++ .../response/response-static-redirect.any.js | 40 + .../response/response-stream-bad-chunk.any.js | 25 + pkg/download/engine/testdata/wpt_harness.js | 113 ++ pkg/download/engine/wpt_fetch_test.go | 120 +++ pkg/download/extension_engine_session.go | 2 - 68 files changed, 7243 insertions(+), 1604 deletions(-) create mode 100644 internal/httpclient/client.go create mode 100644 internal/httpclient/client_test.go create mode 100644 internal/httpclient/httpclienttest/fingerprint_server.go create mode 100644 pkg/download/engine/fetch_integration_test.go create mode 100644 pkg/download/engine/inject/fetch/fetch.js create mode 100644 pkg/download/engine/inject/fetch/module.go delete mode 100644 pkg/download/engine/inject/formdata/module.go delete mode 100644 pkg/download/engine/inject/stream/module_test.go delete mode 100644 pkg/download/engine/inject/xhr/tls_fingerprint.go create mode 100644 pkg/download/engine/inject/xhr/xhr.js delete mode 100644 pkg/download/engine/polyfill/patches/whatwg-fetch+3.6.20.patch delete mode 100644 pkg/download/engine/polyfill/src/fetch/index.js create mode 100644 pkg/download/engine/testdata/wpt/LICENSE.md create mode 100644 pkg/download/engine/testdata/wpt/UPSTREAM.md create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/cors/resources/not-cors-safelisted.json create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/headers/header-setcookie.any.js create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/headers/headers-basic.any.js create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/headers/headers-casing.any.js create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/headers/headers-combine.any.js create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/headers/headers-errors.any.js create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/headers/headers-forbidden-override.any.js create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/headers/headers-no-cors.any.js create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/headers/headers-normalize.any.js create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/headers/headers-record.any.js create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/headers/headers-structure.any.js create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/request/forbidden-method.any.js create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/request/request-clone-readable-stream-body.any.js create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/request/request-constructor-init-body-override.any.js create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/request/request-consume-empty.any.js create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/request/request-consume.any.js create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/request/request-disturbed.any.js create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/request/request-error.any.js create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/request/request-error.js create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/request/request-headers.any.js create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/request/request-init-002.any.js create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/request/request-init-contenttype.any.js create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/request/request-init-priority.any.js create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/request/request-init-stream.any.js create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/request/request-keepalive.any.js create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/request/request-structure.any.js create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/resources/utils.js create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/response/response-consume-empty.any.js create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/response/response-consume-stream.any.js create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/response/response-error.any.js create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/response/response-from-stream.any.js create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/response/response-headers-guard.any.js create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/response/response-init-001.any.js create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/response/response-init-002.any.js create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/response/response-init-contenttype.any.js create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/response/response-static-error.any.js create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/response/response-static-json.any.js create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/response/response-static-redirect.any.js create mode 100644 pkg/download/engine/testdata/wpt/fetch/api/response/response-stream-bad-chunk.any.js create mode 100644 pkg/download/engine/testdata/wpt_harness.js create mode 100644 pkg/download/engine/wpt_fetch_test.go diff --git a/internal/httpclient/client.go b/internal/httpclient/client.go new file mode 100644 index 00000000..5283eb89 --- /dev/null +++ b/internal/httpclient/client.go @@ -0,0 +1,346 @@ +package httpclient + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "net/http" + "net/url" + "strings" + "sync" + "time" + + "github.com/imroc/req/v3" +) + +type ImpersonationMode string + +const ( + ImpersonationNative ImpersonationMode = "native" + ImpersonationFixed ImpersonationMode = "fixed" + ImpersonationAuto ImpersonationMode = "auto" +) + +type Browser string + +const ( + BrowserChrome Browser = "chrome" + BrowserFirefox Browser = "firefox" + BrowserSafari Browser = "safari" +) + +type Options struct { + Client ClientOptions + Transport TransportOptions + Impersonation ImpersonationOptions +} + +type ClientOptions struct { + Jar http.CookieJar + CheckRedirect func(req *http.Request, via []*http.Request) error + Timeout time.Duration +} + +type TransportOptions struct { + Proxy func(*http.Request) (*url.URL, error) + DialContext func(ctx context.Context, network, address string) (net.Conn, error) + TLSClientConfig *tls.Config + TLSHandshakeTimeout time.Duration +} + +type ImpersonationOptions struct { + Mode ImpersonationMode + Browser Browser + Session *ImpersonationSession +} + +type ImpersonationSession struct { + mu sync.RWMutex + selections map[string]impersonationSelection +} + +type impersonationSelection struct { + browser Browser + fallbackDisabled bool +} + +func NewImpersonationSession() *ImpersonationSession { + return &ImpersonationSession{ + selections: make(map[string]impersonationSelection), + } +} + +func (s *ImpersonationSession) load(url string) (impersonationSelection, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + selection, ok := s.selections[url] + return selection, ok +} + +func (s *ImpersonationSession) store(url string, browser Browser) { + s.mu.Lock() + defer s.mu.Unlock() + s.selections[url] = impersonationSelection{browser: browser} +} + +func (s *ImpersonationSession) disableFallback(url string) { + s.mu.Lock() + defer s.mu.Unlock() + s.selections[url] = impersonationSelection{fallbackDisabled: true} +} + +func (s *ImpersonationSession) Clear() { + s.mu.Lock() + defer s.mu.Unlock() + clear(s.selections) +} + +func NewClient(options Options) (*http.Client, error) { + var transport http.RoundTripper + switch options.Impersonation.Mode { + case "", ImpersonationNative: + transport = newNativeTransport(options.Transport) + case ImpersonationFixed: + browserTransport, err := newBrowserTransport(options.Transport, options.Impersonation.Browser) + if err != nil { + return nil, err + } + transport = browserTransport + case ImpersonationAuto: + browserTransports := make(map[Browser]http.RoundTripper, 3) + for _, browser := range []Browser{BrowserChrome, BrowserFirefox, BrowserSafari} { + browserTransport, err := newBrowserTransport(options.Transport, browser) + if err != nil { + return nil, err + } + browserTransports[browser] = browserTransport + } + transport = &adaptiveTransport{ + native: newNativeTransport(options.Transport), + browsers: browserTransports, + jar: options.Client.Jar, + session: options.Impersonation.Session, + } + default: + return nil, fmt.Errorf("unsupported impersonation mode %q", options.Impersonation.Mode) + } + + return &http.Client{ + Transport: transport, + Jar: options.Client.Jar, + CheckRedirect: options.Client.CheckRedirect, + Timeout: options.Client.Timeout, + }, nil +} + +func newNativeTransport(options TransportOptions) *http.Transport { + var tlsConfig *tls.Config + if options.TLSClientConfig != nil { + tlsConfig = options.TLSClientConfig.Clone() + } + return &http.Transport{ + Proxy: options.Proxy, + DialContext: options.DialContext, + TLSClientConfig: tlsConfig, + TLSHandshakeTimeout: options.TLSHandshakeTimeout, + ForceAttemptHTTP2: true, + } +} + +func newBrowserTransport(options TransportOptions, browser Browser) (http.RoundTripper, error) { + client := req.NewClient() + client.SetProxy(options.Proxy) + if options.DialContext != nil { + client.SetDial(options.DialContext) + } + client.SetTLSHandshakeTimeout(options.TLSHandshakeTimeout) + if options.TLSClientConfig != nil { + tlsConfig := options.TLSClientConfig.Clone() + if len(tlsConfig.NextProtos) == 0 { + tlsConfig.NextProtos = []string{"h2", "http/1.1"} + } + client.SetTLSClientConfig(tlsConfig) + } + + switch browser { + case BrowserChrome: + client.ImpersonateChrome() + case BrowserFirefox: + client.ImpersonateFirefox() + case BrowserSafari: + client.ImpersonateSafari() + default: + return nil, fmt.Errorf("unsupported browser %q", browser) + } + + return &defaultHeaderTransport{ + next: client.GetClient().Transport, + defaults: client.Headers.Clone(), + }, nil +} + +type defaultHeaderTransport struct { + next http.RoundTripper + defaults http.Header +} + +func (t *defaultHeaderTransport) RoundTrip(request *http.Request) (*http.Response, error) { + request = request.Clone(request.Context()) + request.Header = request.Header.Clone() + for key, values := range t.defaults { + if len(request.Header.Values(key)) == 0 { + request.Header[key] = append([]string(nil), values...) + } + } + return t.next.RoundTrip(request) +} + +func (t *defaultHeaderTransport) CloseIdleConnections() { + closeIdleConnections(t.next) +} + +type adaptiveTransport struct { + native http.RoundTripper + browsers map[Browser]http.RoundTripper + jar http.CookieJar + session *ImpersonationSession +} + +func (t *adaptiveTransport) RoundTrip(request *http.Request) (*http.Response, error) { + cacheKey := requestCacheKey(request.URL) + if t.session != nil { + if selected, ok := t.session.load(cacheKey); ok { + if selected.fallbackDisabled { + return t.native.RoundTrip(request) + } + return t.browsers[selected.browser].RoundTrip(request) + } + } + + response, err := t.native.RoundTrip(request) + if !shouldFallback(response, err) { + return response, err + } + + retry, ok := cloneRequestForRetry(request) + if !ok { + return response, err + } + if response != nil { + applyResponseCookies(t.jar, request, retry, response) + _ = response.Body.Close() + } + + browser := browserFromUserAgent(request.UserAgent()) + response, err = t.browsers[browser].RoundTrip(retry) + if t.session != nil { + if isSuccessfulFallback(response, err) { + t.session.store(cacheKey, browser) + } else if err == nil && response != nil { + t.session.disableFallback(cacheKey) + } + } + return response, err +} + +func requestCacheKey(requestURL *url.URL) string { + cacheURL := *requestURL + cacheURL.Fragment = "" + cacheURL.RawFragment = "" + return cacheURL.String() +} + +func applyResponseCookies(jar http.CookieJar, request, retry *http.Request, response *http.Response) { + if jar == nil { + return + } + jar.SetCookies(request.URL, response.Cookies()) + jarCookies := jar.Cookies(request.URL) + jarCookieNames := make(map[string]struct{}, len(jarCookies)) + for _, cookie := range jarCookies { + jarCookieNames[cookie.Name] = struct{}{} + } + existingCookies := retry.Cookies() + retry.Header.Del("Cookie") + for _, cookie := range existingCookies { + if _, replaced := jarCookieNames[cookie.Name]; !replaced { + retry.AddCookie(cookie) + } + } + for _, cookie := range jarCookies { + retry.AddCookie(cookie) + } +} + +func (t *adaptiveTransport) CloseIdleConnections() { + closeIdleConnections(t.native) + for _, transport := range t.browsers { + closeIdleConnections(transport) + } +} + +func closeIdleConnections(transport http.RoundTripper) { + if closer, ok := transport.(interface{ CloseIdleConnections() }); ok { + closer.CloseIdleConnections() + } +} + +func shouldFallback(response *http.Response, err error) bool { + if err != nil { + return true + } + if response == nil { + return false + } + return response.StatusCode == http.StatusForbidden || + strings.EqualFold(response.Header.Get("Cf-Mitigated"), "challenge") +} + +func isSuccessfulFallback(response *http.Response, err error) bool { + return err == nil && response != nil && + response.StatusCode >= http.StatusOK && + response.StatusCode < http.StatusBadRequest +} + +func cloneRequestForRetry(request *http.Request) (*http.Request, bool) { + if !isIdempotent(request) { + return nil, false + } + retry := request.Clone(request.Context()) + if request.Body == nil || request.Body == http.NoBody { + return retry, true + } + if request.GetBody == nil { + return nil, false + } + body, err := request.GetBody() + if err != nil { + return nil, false + } + retry.Body = body + return retry, true +} + +func isIdempotent(request *http.Request) bool { + switch request.Method { + case http.MethodGet, http.MethodHead, http.MethodOptions, http.MethodTrace: + return true + } + return request.Header.Get("Idempotency-Key") != "" || + request.Header.Get("X-Idempotency-Key") != "" +} + +func browserFromUserAgent(userAgent string) Browser { + userAgent = strings.ToLower(userAgent) + switch { + case strings.Contains(userAgent, "firefox"): + return BrowserFirefox + case strings.Contains(userAgent, "safari") && + !strings.Contains(userAgent, "chrome") && + !strings.Contains(userAgent, "chromium"): + return BrowserSafari + default: + return BrowserChrome + } +} diff --git a/internal/httpclient/client_test.go b/internal/httpclient/client_test.go new file mode 100644 index 00000000..150fbc62 --- /dev/null +++ b/internal/httpclient/client_test.go @@ -0,0 +1,962 @@ +package httpclient_test + +import ( + "context" + "crypto/tls" + "errors" + "io" + "net" + "net/http" + "net/http/cookiejar" + "net/http/httptest" + "net/url" + "slices" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/GopeedLab/gopeed/internal/httpclient" + "github.com/GopeedLab/gopeed/internal/httpclient/httpclienttest" +) + +func TestFixedChromeUsesBrowserTLSFingerprintAndHTTP2(t *testing.T) { + server := httpclienttest.NewFingerprintServer(t, httpclienttest.FingerprintServerOptions{ + RequiredProfile: httpclienttest.ProfileAnyBrowser, + }) + + client, err := httpclient.NewClient(httpclient.Options{ + Transport: httpclient.TransportOptions{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + Impersonation: httpclient.ImpersonationOptions{ + Mode: httpclient.ImpersonationFixed, + Browser: httpclient.BrowserChrome, + }, + }) + if err != nil { + t.Fatalf("NewClient() error = %v", err) + } + + response, err := client.Get(server.URL) + if err != nil { + t.Fatalf("Get() error = %v", err) + } + defer response.Body.Close() + + if response.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want %d", response.StatusCode, http.StatusOK) + } + if response.ProtoMajor != 2 { + t.Fatalf("protocol = %q, want HTTP/2", response.Proto) + } +} + +func TestAutoFallsBackFromNativeToBrowserFingerprint(t *testing.T) { + server := httpclienttest.NewFingerprintServer(t, httpclienttest.FingerprintServerOptions{ + RequiredProfile: httpclienttest.ProfileAnyBrowser, + }) + + client, err := httpclient.NewClient(httpclient.Options{ + Transport: httpclient.TransportOptions{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + Impersonation: httpclient.ImpersonationOptions{ + Mode: httpclient.ImpersonationAuto, + }, + }) + if err != nil { + t.Fatalf("NewClient() error = %v", err) + } + + request, err := http.NewRequest(http.MethodGet, server.URL, nil) + if err != nil { + t.Fatalf("NewRequest() error = %v", err) + } + request.Header.Set("User-Agent", "Mozilla/5.0 Chrome/120.0.0.0 Safari/537.36") + + response, err := client.Do(request) + if err != nil { + t.Fatalf("Do() error = %v", err) + } + defer response.Body.Close() + + if response.ProtoMajor != 2 { + t.Fatalf("protocol = %q, want HTTP/2", response.Proto) + } + if got, want := server.Profiles(), []httpclienttest.Profile{ + httpclienttest.ProfileNative, + httpclienttest.ProfileChrome, + }; !slices.Equal(got, want) { + t.Fatalf("ClientHello profiles = %v, want %v", got, want) + } +} + +func TestAutoChoosesBrowserFingerprintFromUserAgent(t *testing.T) { + tests := []struct { + name string + userAgent string + wantBrowser httpclient.Browser + }{ + { + name: "chrome", + userAgent: "Mozilla/5.0 Chrome/120.0.0.0 Safari/537.36", + wantBrowser: httpclient.BrowserChrome, + }, + { + name: "firefox", + userAgent: "Mozilla/5.0 Firefox/120.0", + wantBrowser: httpclient.BrowserFirefox, + }, + { + name: "safari", + userAgent: "Mozilla/5.0 Version/16.6 Safari/605.1.15", + wantBrowser: httpclient.BrowserSafari, + }, + { + name: "unknown defaults to chrome", + userAgent: "Gopeed/1.0", + wantBrowser: httpclient.BrowserChrome, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + server := httpclienttest.NewFingerprintServer(t, httpclienttest.FingerprintServerOptions{ + RequiredProfile: httpclienttest.Profile(test.wantBrowser), + }) + client, err := httpclient.NewClient(httpclient.Options{ + Transport: httpclient.TransportOptions{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + Impersonation: httpclient.ImpersonationOptions{Mode: httpclient.ImpersonationAuto}, + }) + if err != nil { + t.Fatalf("NewClient() error = %v", err) + } + + request, err := http.NewRequest(http.MethodGet, server.URL, nil) + if err != nil { + t.Fatalf("NewRequest() error = %v", err) + } + request.Header.Set("User-Agent", test.userAgent) + response, err := client.Do(request) + if err != nil { + t.Fatalf("Do() error = %v", err) + } + response.Body.Close() + + if response.ProtoMajor != 2 { + t.Fatalf("protocol = %q, want HTTP/2", response.Proto) + } + if got, want := server.Profiles(), []httpclienttest.Profile{ + httpclienttest.ProfileNative, + httpclienttest.Profile(test.wantBrowser), + }; !slices.Equal(got, want) { + t.Fatalf("profiles = %v, want %v", got, want) + } + }) + } +} + +func TestAutoFallsBackOnForbiddenAndPinsBrowserPerURL(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + requests.Add(1) + if request.Header.Get("Sec-Ch-Ua") == "" { + http.Error(w, "browser required", http.StatusForbidden) + return + } + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(server.Close) + + client, err := httpclient.NewClient(httpclient.Options{ + Impersonation: httpclient.ImpersonationOptions{ + Mode: httpclient.ImpersonationAuto, + Session: httpclient.NewImpersonationSession(), + }, + }) + if err != nil { + t.Fatalf("NewClient() error = %v", err) + } + + for range 2 { + request, err := http.NewRequest(http.MethodGet, server.URL, nil) + if err != nil { + t.Fatalf("NewRequest() error = %v", err) + } + request.Header.Set("User-Agent", "Mozilla/5.0 Chrome/120.0.0.0 Safari/537.36") + response, err := client.Do(request) + if err != nil { + t.Fatalf("Do() error = %v", err) + } + response.Body.Close() + if response.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want %d", response.StatusCode, http.StatusOK) + } + } + + if requests.Load() != 3 { + t.Fatalf("requests = %d, want 3 (native + fallback + pinned)", requests.Load()) + } +} + +func TestAutoDisablesFallbackWhenBrowserReturnsHTTPFailure(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + requests.Add(1) + if request.Header.Get("Sec-Ch-Ua") == "" { + w.WriteHeader(http.StatusForbidden) + return + } + w.WriteHeader(http.StatusNotFound) + })) + t.Cleanup(server.Close) + + client, err := httpclient.NewClient(httpclient.Options{ + Impersonation: httpclient.ImpersonationOptions{ + Mode: httpclient.ImpersonationAuto, + Session: httpclient.NewImpersonationSession(), + }, + }) + if err != nil { + t.Fatalf("NewClient() error = %v", err) + } + + wantStatuses := []int{http.StatusNotFound, http.StatusForbidden} + for _, wantStatus := range wantStatuses { + response, err := client.Get(server.URL) + if err != nil { + t.Fatalf("Get() error = %v", err) + } + response.Body.Close() + if response.StatusCode != wantStatus { + t.Fatalf("status = %d, want %d", response.StatusCode, wantStatus) + } + } + + if requests.Load() != 3 { + t.Fatalf("requests = %d, want 3 (native + one fallback + native)", requests.Load()) + } +} + +func TestAutoDoesNotDisableFallbackAfterTransportFailure(t *testing.T) { + server := httpclienttest.NewFingerprintServer(t, httpclienttest.FingerprintServerOptions{ + RequiredProfile: httpclienttest.Profile("unsupported"), + }) + client, err := httpclient.NewClient(httpclient.Options{ + Transport: httpclient.TransportOptions{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + Impersonation: httpclient.ImpersonationOptions{ + Mode: httpclient.ImpersonationAuto, + Session: httpclient.NewImpersonationSession(), + }, + }) + if err != nil { + t.Fatalf("NewClient() error = %v", err) + } + + for range 2 { + if _, err := client.Get(server.URL); err == nil { + t.Fatal("Get() error = nil, want TLS failure") + } + } + if got, want := server.Profiles(), []httpclienttest.Profile{ + httpclienttest.ProfileNative, + httpclienttest.ProfileChrome, + httpclienttest.ProfileNative, + httpclienttest.ProfileChrome, + }; !slices.Equal(got, want) { + t.Fatalf("ClientHello profiles = %v, want %v", got, want) + } +} + +func TestAutoWithoutSessionDoesNotCacheSelectedBrowser(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + requests.Add(1) + if request.Header.Get("Sec-Ch-Ua") == "" { + w.WriteHeader(http.StatusForbidden) + return + } + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(server.Close) + + client, err := httpclient.NewClient(httpclient.Options{ + Impersonation: httpclient.ImpersonationOptions{Mode: httpclient.ImpersonationAuto}, + }) + if err != nil { + t.Fatalf("NewClient() error = %v", err) + } + for range 2 { + response, err := client.Get(server.URL) + if err != nil { + t.Fatalf("Get() error = %v", err) + } + response.Body.Close() + } + + if requests.Load() != 4 { + t.Fatalf("requests = %d, want 4 (native + fallback without caching)", requests.Load()) + } +} + +func TestAutoSharesSelectedBrowserAcrossClientsInSession(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + requests.Add(1) + if request.Header.Get("Sec-Ch-Ua") == "" { + w.WriteHeader(http.StatusForbidden) + return + } + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(server.Close) + session := httpclient.NewImpersonationSession() + + for range 2 { + client, err := httpclient.NewClient(httpclient.Options{ + Impersonation: httpclient.ImpersonationOptions{ + Mode: httpclient.ImpersonationAuto, + Session: session, + }, + }) + if err != nil { + t.Fatalf("NewClient() error = %v", err) + } + request, err := http.NewRequest(http.MethodGet, server.URL, nil) + if err != nil { + t.Fatalf("NewRequest() error = %v", err) + } + request.Header.Set("User-Agent", "Mozilla/5.0 Chrome/120.0.0.0 Safari/537.36") + response, err := client.Do(request) + if err != nil { + t.Fatalf("Do() error = %v", err) + } + response.Body.Close() + } + + if requests.Load() != 3 { + t.Fatalf("requests = %d, want 3 (native + fallback + session pinned)", requests.Load()) + } +} + +func TestAutoDoesNotShareSelectedBrowserAcrossSessions(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + requests.Add(1) + if request.Header.Get("Sec-Ch-Ua") == "" { + w.WriteHeader(http.StatusForbidden) + return + } + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(server.Close) + + for range 2 { + client, err := httpclient.NewClient(httpclient.Options{ + Impersonation: httpclient.ImpersonationOptions{ + Mode: httpclient.ImpersonationAuto, + Session: httpclient.NewImpersonationSession(), + }, + }) + if err != nil { + t.Fatalf("NewClient() error = %v", err) + } + response, err := client.Get(server.URL) + if err != nil { + t.Fatalf("Get() error = %v", err) + } + response.Body.Close() + } + + if requests.Load() != 4 { + t.Fatalf("requests = %d, want 4 (native + fallback per session)", requests.Load()) + } +} + +func TestAutoCachesSelectedBrowserByFullURL(t *testing.T) { + var publicBrowserRequests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + browserRequest := request.Header.Get("Sec-Ch-Ua") != "" + switch request.URL.Query().Get("access") { + case "protected": + if !browserRequest { + w.WriteHeader(http.StatusForbidden) + return + } + case "public": + if browserRequest { + publicBrowserRequests.Add(1) + } + } + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(server.Close) + + client, err := httpclient.NewClient(httpclient.Options{ + Impersonation: httpclient.ImpersonationOptions{ + Mode: httpclient.ImpersonationAuto, + Session: httpclient.NewImpersonationSession(), + }, + }) + if err != nil { + t.Fatalf("NewClient() error = %v", err) + } + + for range 2 { + response, err := client.Get(server.URL + "/file?access=protected") + if err != nil { + t.Fatalf("protected Get() error = %v", err) + } + response.Body.Close() + } + response, err := client.Get(server.URL + "/file?access=public") + if err != nil { + t.Fatalf("public Get() error = %v", err) + } + response.Body.Close() + + if publicBrowserRequests.Load() != 0 { + t.Fatalf("public browser requests = %d, want 0", publicBrowserRequests.Load()) + } +} + +func TestImpersonationSessionClearRemovesSelections(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + requests.Add(1) + if request.Header.Get("Sec-Ch-Ua") == "" { + w.WriteHeader(http.StatusForbidden) + return + } + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(server.Close) + + session := httpclient.NewImpersonationSession() + client, err := httpclient.NewClient(httpclient.Options{ + Impersonation: httpclient.ImpersonationOptions{ + Mode: httpclient.ImpersonationAuto, + Session: session, + }, + }) + if err != nil { + t.Fatalf("NewClient() error = %v", err) + } + for range 2 { + response, err := client.Get(server.URL) + if err != nil { + t.Fatalf("Get() error = %v", err) + } + response.Body.Close() + session.Clear() + } + + if requests.Load() != 4 { + t.Fatalf("requests = %d, want 4 (native + fallback after each clear)", requests.Load()) + } +} + +func TestGlobalAutoSelectionDoesNotAffectNativeMode(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + if request.Header.Get("Sec-Ch-Ua") == "" { + w.WriteHeader(http.StatusForbidden) + return + } + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(server.Close) + + autoClient, err := httpclient.NewClient(httpclient.Options{ + Impersonation: httpclient.ImpersonationOptions{Mode: httpclient.ImpersonationAuto}, + }) + if err != nil { + t.Fatalf("NewClient(auto) error = %v", err) + } + autoResponse, err := autoClient.Get(server.URL) + if err != nil { + t.Fatalf("auto Get() error = %v", err) + } + autoResponse.Body.Close() + if autoResponse.StatusCode != http.StatusOK { + t.Fatalf("auto status = %d, want %d", autoResponse.StatusCode, http.StatusOK) + } + + nativeClient, err := httpclient.NewClient(httpclient.Options{ + Impersonation: httpclient.ImpersonationOptions{Mode: httpclient.ImpersonationNative}, + }) + if err != nil { + t.Fatalf("NewClient(native) error = %v", err) + } + nativeResponse, err := nativeClient.Get(server.URL) + if err != nil { + t.Fatalf("native Get() error = %v", err) + } + nativeResponse.Body.Close() + if nativeResponse.StatusCode != http.StatusForbidden { + t.Fatalf("native status = %d, want %d", nativeResponse.StatusCode, http.StatusForbidden) + } +} + +func TestAutoAppliesFallbackResponseCookiesBeforeRetry(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + if request.Header.Get("Sec-Ch-Ua") == "" { + http.SetCookie(w, &http.Cookie{Name: "challenge", Value: "passed", Path: "/"}) + w.WriteHeader(http.StatusForbidden) + return + } + cookie, err := request.Cookie("challenge") + if err != nil || cookie.Value != "passed" { + w.WriteHeader(http.StatusForbidden) + return + } + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(server.Close) + + jar, err := cookiejar.New(nil) + if err != nil { + t.Fatalf("cookiejar.New() error = %v", err) + } + client, err := httpclient.NewClient(httpclient.Options{ + Client: httpclient.ClientOptions{Jar: jar}, + Impersonation: httpclient.ImpersonationOptions{Mode: httpclient.ImpersonationAuto}, + }) + if err != nil { + t.Fatalf("NewClient() error = %v", err) + } + response, err := client.Get(server.URL) + if err != nil { + t.Fatalf("Get() error = %v", err) + } + response.Body.Close() + if response.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want %d", response.StatusCode, http.StatusOK) + } +} + +func TestAutoDoesNotRetryRequestWithNonReplayableBody(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + w.WriteHeader(http.StatusForbidden) + })) + t.Cleanup(server.Close) + + client, err := httpclient.NewClient(httpclient.Options{ + Impersonation: httpclient.ImpersonationOptions{Mode: httpclient.ImpersonationAuto}, + }) + if err != nil { + t.Fatalf("NewClient() error = %v", err) + } + bodyReader, bodyWriter := io.Pipe() + t.Cleanup(func() { + bodyReader.Close() + bodyWriter.Close() + }) + go func() { + _, _ = bodyWriter.Write([]byte("payload")) + _ = bodyWriter.Close() + }() + request, err := http.NewRequest(http.MethodPost, server.URL, bodyReader) + if err != nil { + t.Fatalf("NewRequest() error = %v", err) + } + + response, err := client.Do(request) + if err != nil { + t.Fatalf("Do() error = %v", err) + } + response.Body.Close() + if requests.Load() != 1 { + t.Fatalf("requests = %d, want 1", requests.Load()) + } +} + +func TestAutoDoesNotRetryReplayablePost(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + w.WriteHeader(http.StatusForbidden) + })) + t.Cleanup(server.Close) + + client, err := httpclient.NewClient(httpclient.Options{ + Impersonation: httpclient.ImpersonationOptions{Mode: httpclient.ImpersonationAuto}, + }) + if err != nil { + t.Fatalf("NewClient() error = %v", err) + } + request, err := http.NewRequest(http.MethodPost, server.URL, strings.NewReader("payload")) + if err != nil { + t.Fatalf("NewRequest() error = %v", err) + } + + response, err := client.Do(request) + if err != nil { + t.Fatalf("Do() error = %v", err) + } + response.Body.Close() + if requests.Load() != 1 { + t.Fatalf("requests = %d, want 1", requests.Load()) + } +} + +func TestNativeUsesStandardTransportWithHTTP2(t *testing.T) { + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + server.EnableHTTP2 = true + server.StartTLS() + t.Cleanup(server.Close) + + client, err := httpclient.NewClient(httpclient.Options{ + Transport: httpclient.TransportOptions{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + Impersonation: httpclient.ImpersonationOptions{ + Mode: httpclient.ImpersonationNative, + }, + }) + if err != nil { + t.Fatalf("NewClient() error = %v", err) + } + + response, err := client.Get(server.URL) + if err != nil { + t.Fatalf("Get() error = %v", err) + } + defer response.Body.Close() + + if response.ProtoMajor != 2 { + t.Fatalf("protocol = %q, want HTTP/2", response.Proto) + } +} + +func TestClientOptionsPreserveCookiesAcrossRedirects(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/start": + http.SetCookie(w, &http.Cookie{Name: "session", Value: "gopeed", Path: "/"}) + http.Redirect(w, request, "/download", http.StatusFound) + case "/download": + cookie, err := request.Cookie("session") + if err != nil || cookie.Value != "gopeed" { + http.Error(w, "cookie missing", http.StatusBadRequest) + return + } + w.WriteHeader(http.StatusOK) + default: + http.NotFound(w, request) + } + })) + t.Cleanup(server.Close) + + jar, err := cookiejar.New(nil) + if err != nil { + t.Fatalf("cookiejar.New() error = %v", err) + } + var redirects atomic.Int32 + client, err := httpclient.NewClient(httpclient.Options{ + Client: httpclient.ClientOptions{ + Jar: jar, + CheckRedirect: func(_ *http.Request, _ []*http.Request) error { + redirects.Add(1) + return nil + }, + }, + Impersonation: httpclient.ImpersonationOptions{Mode: httpclient.ImpersonationNative}, + }) + if err != nil { + t.Fatalf("NewClient() error = %v", err) + } + + response, err := client.Get(server.URL + "/start") + if err != nil { + t.Fatalf("Get() error = %v", err) + } + defer response.Body.Close() + + if response.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want %d", response.StatusCode, http.StatusOK) + } + if redirects.Load() != 1 { + t.Fatalf("redirect calls = %d, want 1", redirects.Load()) + } +} + +func TestClientTimeoutCancelsRequest(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, request *http.Request) { + <-request.Context().Done() + })) + t.Cleanup(server.Close) + + client, err := httpclient.NewClient(httpclient.Options{ + Client: httpclient.ClientOptions{Timeout: 50 * time.Millisecond}, + Impersonation: httpclient.ImpersonationOptions{ + Mode: httpclient.ImpersonationNative, + }, + }) + if err != nil { + t.Fatalf("NewClient() error = %v", err) + } + + _, err = client.Get(server.URL) + if err == nil { + t.Fatal("Get() error = nil, want timeout") + } + var networkError net.Error + if !errors.As(err, &networkError) || !networkError.Timeout() { + t.Fatalf("Get() error = %v, want net.Error timeout", err) + } +} + +func TestTransportOptionsUseProxyInNativeAndFixedModes(t *testing.T) { + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + if request.URL.Host != "download.example" { + http.Error(w, "unexpected target", http.StatusBadRequest) + return + } + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(proxy.Close) + proxyURL, err := url.Parse(proxy.URL) + if err != nil { + t.Fatalf("url.Parse() error = %v", err) + } + + tests := []struct { + name string + impersonation httpclient.ImpersonationOptions + }{ + { + name: "native", + impersonation: httpclient.ImpersonationOptions{Mode: httpclient.ImpersonationNative}, + }, + { + name: "fixed chrome", + impersonation: httpclient.ImpersonationOptions{ + Mode: httpclient.ImpersonationFixed, + Browser: httpclient.BrowserChrome, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + client, err := httpclient.NewClient(httpclient.Options{ + Transport: httpclient.TransportOptions{Proxy: http.ProxyURL(proxyURL)}, + Impersonation: test.impersonation, + }) + if err != nil { + t.Fatalf("NewClient() error = %v", err) + } + + response, err := client.Get("http://download.example/file") + if err != nil { + t.Fatalf("Get() error = %v", err) + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want %d", response.StatusCode, http.StatusOK) + } + }) + } +} + +func TestTransportOptionsUseDialContextInNativeAndFixedModes(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(server.Close) + + tests := []struct { + name string + impersonation httpclient.ImpersonationOptions + }{ + { + name: "native", + impersonation: httpclient.ImpersonationOptions{Mode: httpclient.ImpersonationNative}, + }, + { + name: "fixed chrome", + impersonation: httpclient.ImpersonationOptions{ + Mode: httpclient.ImpersonationFixed, + Browser: httpclient.BrowserChrome, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var calls atomic.Int32 + dialer := &net.Dialer{} + client, err := httpclient.NewClient(httpclient.Options{ + Transport: httpclient.TransportOptions{ + DialContext: func(ctx context.Context, network, address string) (net.Conn, error) { + calls.Add(1) + return dialer.DialContext(ctx, network, address) + }, + }, + Impersonation: test.impersonation, + }) + if err != nil { + t.Fatalf("NewClient() error = %v", err) + } + + response, err := client.Get(server.URL) + if err != nil { + t.Fatalf("Get() error = %v", err) + } + response.Body.Close() + if calls.Load() == 0 { + t.Fatal("DialContext() was not called") + } + }) + } +} + +func TestTransportOptionsApplyTLSHandshakeTimeoutInNativeAndFixedModes(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("net.Listen() error = %v", err) + } + closed := make(chan struct{}) + t.Cleanup(func() { + close(closed) + listener.Close() + }) + go func() { + for { + connection, err := listener.Accept() + if err != nil { + return + } + go func() { + <-closed + connection.Close() + }() + } + }() + + tests := []struct { + name string + impersonation httpclient.ImpersonationOptions + }{ + { + name: "native", + impersonation: httpclient.ImpersonationOptions{Mode: httpclient.ImpersonationNative}, + }, + { + name: "fixed chrome", + impersonation: httpclient.ImpersonationOptions{ + Mode: httpclient.ImpersonationFixed, + Browser: httpclient.BrowserChrome, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + client, err := httpclient.NewClient(httpclient.Options{ + Client: httpclient.ClientOptions{Timeout: time.Second}, + Transport: httpclient.TransportOptions{ + TLSHandshakeTimeout: 50 * time.Millisecond, + }, + Impersonation: test.impersonation, + }) + if err != nil { + t.Fatalf("NewClient() error = %v", err) + } + + started := time.Now() + _, err = client.Get("https://" + listener.Addr().String()) + if err == nil { + t.Fatal("Get() error = nil, want TLS handshake timeout") + } + if elapsed := time.Since(started); elapsed >= 500*time.Millisecond { + t.Fatalf("TLS handshake took %s, want less than 500ms", elapsed) + } + }) + } +} + +func TestFixedBrowserAddsProfileHeadersWithoutOverwritingExplicitHeaders(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + w.Header().Set("Seen-User-Agent", request.UserAgent()) + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(server.Close) + + client, err := httpclient.NewClient(httpclient.Options{ + Impersonation: httpclient.ImpersonationOptions{ + Mode: httpclient.ImpersonationFixed, + Browser: httpclient.BrowserChrome, + }, + }) + if err != nil { + t.Fatalf("NewClient() error = %v", err) + } + + response, err := client.Get(server.URL) + if err != nil { + t.Fatalf("Get() error = %v", err) + } + response.Body.Close() + if userAgent := response.Header.Get("Seen-User-Agent"); !strings.Contains(userAgent, "Chrome/120") { + t.Fatalf("profile User-Agent = %q, want Chrome/120", userAgent) + } + + request, err := http.NewRequest(http.MethodGet, server.URL, nil) + if err != nil { + t.Fatalf("NewRequest() error = %v", err) + } + request.Header.Set("User-Agent", "Gopeed-Custom/1.0") + response, err = client.Do(request) + if err != nil { + t.Fatalf("Do() error = %v", err) + } + response.Body.Close() + if userAgent := response.Header.Get("Seen-User-Agent"); userAgent != "Gopeed-Custom/1.0" { + t.Fatalf("explicit User-Agent = %q, want Gopeed-Custom/1.0", userAgent) + } +} + +func TestFixedClientCloseIdleConnectionsReachesBrowserTransport(t *testing.T) { + var connections atomic.Int32 + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + server.Config.ConnState = func(_ net.Conn, state http.ConnState) { + if state == http.StateNew { + connections.Add(1) + } + } + server.Start() + t.Cleanup(server.Close) + + client, err := httpclient.NewClient(httpclient.Options{ + Impersonation: httpclient.ImpersonationOptions{ + Mode: httpclient.ImpersonationFixed, + Browser: httpclient.BrowserChrome, + }, + }) + if err != nil { + t.Fatalf("NewClient() error = %v", err) + } + for attempt := range 2 { + response, err := client.Get(server.URL) + if err != nil { + t.Fatalf("Get() error = %v", err) + } + _, _ = io.Copy(io.Discard, response.Body) + response.Body.Close() + if attempt == 0 { + client.CloseIdleConnections() + } + } + + if connections.Load() != 2 { + t.Fatalf("connections = %d, want 2", connections.Load()) + } +} diff --git a/internal/httpclient/httpclienttest/fingerprint_server.go b/internal/httpclient/httpclienttest/fingerprint_server.go new file mode 100644 index 00000000..fd4d32ca --- /dev/null +++ b/internal/httpclient/httpclienttest/fingerprint_server.go @@ -0,0 +1,100 @@ +package httpclienttest + +import ( + "crypto/tls" + "errors" + "io" + "log" + "net/http" + "net/http/httptest" + "sync" + "testing" +) + +type Profile string + +const ( + ProfileNative Profile = "native" + ProfileChrome Profile = "chrome" + ProfileFirefox Profile = "firefox" + ProfileSafari Profile = "safari" + ProfileAnyBrowser Profile = "browser" +) + +type FingerprintServerOptions struct { + RequiredProfile Profile + Handler http.Handler +} + +type FingerprintServer struct { + *httptest.Server + + mu sync.Mutex + profiles []Profile +} + +func NewFingerprintServer(t testing.TB, options FingerprintServerOptions) *FingerprintServer { + t.Helper() + + handler := options.Handler + if handler == nil { + handler = http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + }) + } + server := &FingerprintServer{ + Server: httptest.NewUnstartedServer(handler), + } + server.EnableHTTP2 = true + server.Config.ErrorLog = log.New(io.Discard, "", 0) + server.TLS = &tls.Config{ + GetConfigForClient: func(hello *tls.ClientHelloInfo) (*tls.Config, error) { + profile := classifyProfile(hello.CipherSuites) + server.mu.Lock() + server.profiles = append(server.profiles, profile) + server.mu.Unlock() + if profileMatches(profile, options.RequiredProfile) { + return nil, nil + } + return nil, errors.New("required TLS fingerprint was not presented") + }, + } + server.StartTLS() + t.Cleanup(server.Close) + return server +} + +func (s *FingerprintServer) Profiles() []Profile { + s.mu.Lock() + defer s.mu.Unlock() + return append([]Profile(nil), s.profiles...) +} + +func profileMatches(actual, required Profile) bool { + if required == ProfileAnyBrowser { + return actual != ProfileNative + } + return actual == required +} + +func classifyProfile(cipherSuites []uint16) Profile { + if len(cipherSuites) >= 5 && isGREASE(cipherSuites[0]) { + switch cipherSuites[4] { + case tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256: + return ProfileChrome + case tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384: + return ProfileSafari + } + } + if len(cipherSuites) >= 3 && + cipherSuites[0] == tls.TLS_AES_128_GCM_SHA256 && + cipherSuites[1] == tls.TLS_CHACHA20_POLY1305_SHA256 && + cipherSuites[2] == tls.TLS_AES_256_GCM_SHA384 { + return ProfileFirefox + } + return ProfileNative +} + +func isGREASE(value uint16) bool { + return value&0x0f0f == 0x0a0a && byte(value) == byte(value>>8) +} diff --git a/internal/protocol/http/fetcher.go b/internal/protocol/http/fetcher.go index a9ed14ad..25827b37 100644 --- a/internal/protocol/http/fetcher.go +++ b/internal/protocol/http/fetcher.go @@ -17,6 +17,7 @@ import ( "github.com/GopeedLab/gopeed/internal/controller" "github.com/GopeedLab/gopeed/internal/fetcher" + "github.com/GopeedLab/gopeed/internal/httpclient" "github.com/GopeedLab/gopeed/pkg/base" fhttp "github.com/GopeedLab/gopeed/pkg/protocol/http" "github.com/xiaoqidun/setft" @@ -213,6 +214,8 @@ type Fetcher struct { config *config doneCh chan error + impersonationSession *httpclient.ImpersonationSession + meta *fetcher.FetcherMeta // State machine @@ -277,6 +280,9 @@ func (f *Fetcher) Setup(ctl *controller.Controller) { f.meta = &fetcher.FetcherMeta{} } f.ctl.GetConfig(&f.config) + if f.impersonationSession == nil { + f.impersonationSession = httpclient.NewImpersonationSession() + } f.resolvedCh = make(chan struct{}) f.primaryReadyCh = make(chan struct{}) @@ -1713,7 +1719,11 @@ func (f *Fetcher) Pause() error { } func (f *Fetcher) Close() error { - return f.Pause() + err := f.Pause() + if f.impersonationSession != nil { + f.impersonationSession.Clear() + } + return err } func (f *Fetcher) Meta() *fetcher.FetcherMeta { diff --git a/internal/protocol/http/fetcher_test.go b/internal/protocol/http/fetcher_test.go index ac086180..fb1386e0 100644 --- a/internal/protocol/http/fetcher_test.go +++ b/internal/protocol/http/fetcher_test.go @@ -5,14 +5,17 @@ import ( "fmt" "net" gohttp "net/http" + "net/http/httptest" "net/url" "os" "strings" + "sync/atomic" "testing" "time" "github.com/GopeedLab/gopeed/internal/controller" "github.com/GopeedLab/gopeed/internal/fetcher" + "github.com/GopeedLab/gopeed/internal/httpclient/httpclienttest" "github.com/GopeedLab/gopeed/internal/test" "github.com/GopeedLab/gopeed/pkg/base" "github.com/GopeedLab/gopeed/pkg/protocol/http" @@ -259,6 +262,79 @@ func TestFetcher_ResolveWithInvalidHeader(t *testing.T) { } } +func TestFetcher_ResolveAutomaticallyFallsBackToBrowserFingerprint(t *testing.T) { + server := httpclienttest.NewFingerprintServer(t, httpclienttest.FingerprintServerOptions{ + RequiredProfile: httpclienttest.ProfileAnyBrowser, + Handler: gohttp.HandlerFunc(func(w gohttp.ResponseWriter, _ *gohttp.Request) { + w.Header().Set(base.HttpHeaderContentLength, "4") + w.Header().Set(base.HttpHeaderContentDisposition, `attachment; filename="file.bin"`) + _, _ = w.Write([]byte("data")) + }), + }) + + fetcher := buildFetcher() + defer fetcher.Pause() + err := fetcher.Resolve(&base.Request{ + URL: server.URL + "/file.bin", + SkipVerifyCert: true, + }, &base.Options{ + Name: test.DownloadName, + Path: test.Dir, + }) + if err != nil { + t.Fatalf("Resolve() error = %v", err) + } + if fetcher.meta.Res == nil || len(fetcher.meta.Res.Files) != 1 { + t.Fatalf("resource = %+v, want one file", fetcher.meta.Res) + } + if fetcher.meta.Res.Files[0].Name != "file.bin" { + t.Fatalf("file name = %q, want file.bin", fetcher.meta.Res.Files[0].Name) + } +} + +func TestFetcherSharesAndClearsImpersonationSession(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(gohttp.HandlerFunc(func(w gohttp.ResponseWriter, request *gohttp.Request) { + requests.Add(1) + if request.Header.Get("Sec-Ch-Ua") == "" { + w.WriteHeader(gohttp.StatusForbidden) + return + } + w.WriteHeader(gohttp.StatusOK) + })) + defer server.Close() + + fetcher := buildFetcher() + fetcher.meta.Req = &base.Request{URL: server.URL} + + for range 2 { + client := fetcher.buildClient() + response, err := client.Get(server.URL) + if err != nil { + t.Fatalf("Get() error = %v", err) + } + response.Body.Close() + client.CloseIdleConnections() + } + if requests.Load() != 3 { + t.Fatalf("requests before completion = %d, want 3", requests.Load()) + } + + if err := fetcher.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + client := fetcher.buildClient() + response, err := client.Get(server.URL) + if err != nil { + t.Fatalf("Get() after close error = %v", err) + } + response.Body.Close() + client.CloseIdleConnections() + if requests.Load() != 5 { + t.Fatalf("requests after close = %d, want 5", requests.Load()) + } +} + func testResolve(startTestServer func() net.Listener, path string, t *testing.T, wantFn func(error) (*base.Resource, error)) { listener := startTestServer() defer listener.Close() diff --git a/internal/protocol/http/helper.go b/internal/protocol/http/helper.go index 25b93da1..ae0395e1 100644 --- a/internal/protocol/http/helper.go +++ b/internal/protocol/http/helper.go @@ -16,6 +16,7 @@ import ( "time" "unicode/utf8" + "github.com/GopeedLab/gopeed/internal/httpclient" "github.com/GopeedLab/gopeed/pkg/base" fhttp "github.com/GopeedLab/gopeed/pkg/protocol/http" "github.com/GopeedLab/gopeed/pkg/util" @@ -216,21 +217,30 @@ func (f *Fetcher) buildFastFailClient() *http.Client { // buildClientWithTimeout creates an HTTP client with the specified connection timeout. func (f *Fetcher) buildClientWithTimeout(timeout time.Duration) *http.Client { - transport := &http.Transport{ - DialContext: (&net.Dialer{ - Timeout: timeout, - }).DialContext, - Proxy: f.ctl.GetProxy(f.meta.Req.Proxy), - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: f.meta.Req.SkipVerifyCert, - }, - TLSHandshakeTimeout: timeout, - } jar, _ := cookiejar.New(nil) - return &http.Client{ - Transport: transport, - Jar: jar, + client, err := httpclient.NewClient(httpclient.Options{ + Client: httpclient.ClientOptions{ + Jar: jar, + }, + Transport: httpclient.TransportOptions{ + DialContext: (&net.Dialer{ + Timeout: timeout, + }).DialContext, + Proxy: f.ctl.GetProxy(f.meta.Req.Proxy), + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: f.meta.Req.SkipVerifyCert, + }, + TLSHandshakeTimeout: timeout, + }, + Impersonation: httpclient.ImpersonationOptions{ + Mode: httpclient.ImpersonationAuto, + Session: f.impersonationSession, + }, + }) + if err != nil { + panic(fmt.Sprintf("build HTTP client: %v", err)) } + return client } // ============================================================================ diff --git a/pkg/download/engine/engine.go b/pkg/download/engine/engine.go index b78db85a..fb93662a 100644 --- a/pkg/download/engine/engine.go +++ b/pkg/download/engine/engine.go @@ -8,8 +8,8 @@ import ( "github.com/GopeedLab/gopeed/pkg/base" gojaerror "github.com/GopeedLab/gopeed/pkg/download/engine/inject/error" + fetchapi "github.com/GopeedLab/gopeed/pkg/download/engine/inject/fetch" "github.com/GopeedLab/gopeed/pkg/download/engine/inject/file" - "github.com/GopeedLab/gopeed/pkg/download/engine/inject/formdata" "github.com/GopeedLab/gopeed/pkg/download/engine/inject/stream" "github.com/GopeedLab/gopeed/pkg/download/engine/inject/vm" "github.com/GopeedLab/gopeed/pkg/download/engine/inject/xhr" @@ -24,6 +24,10 @@ var polyfillScript string type Engine struct { loop *eventloop.EventLoop + closeOnce sync.Once + cleanupMu sync.Mutex + cleanups []func() + Runtime *goja.Runtime } @@ -133,7 +137,25 @@ func (e *Engine) runOnLoop(fn func(runtime *goja.Runtime) (goja.Value, error)) ( } func (e *Engine) Close() { - e.loop.Terminate() + e.closeOnce.Do(func() { + e.cleanupMu.Lock() + cleanups := e.cleanups + e.cleanups = nil + e.cleanupMu.Unlock() + for _, cleanup := range cleanups { + cleanup() + } + e.loop.Terminate() + }) +} + +func (e *Engine) addCleanup(cleanup func()) { + if cleanup == nil { + return + } + e.cleanupMu.Lock() + e.cleanups = append(e.cleanups, cleanup) + e.cleanupMu.Unlock() } type Config struct { @@ -163,12 +185,6 @@ func NewEngine(cfg *Config) *Engine { if err := file.Enable(runtime); err != nil { return } - if err := formdata.Enable(runtime); err != nil { - return - } - if err := xhr.Enable(runtime, cfg.ProxyConfig.ToHandler()); err != nil { - return - } if _, err := runtime.RunString(polyfillScript); err != nil { return } @@ -187,6 +203,15 @@ func NewEngine(cfg *Config) *Engine { if err := stream.Enable(runtime, loop, cfg.StreamConfig); err != nil { return } + if err := fetchapi.Enable(runtime, loop, &fetchapi.Config{ + ProxyHandler: cfg.ProxyConfig.ToHandler(), + RegisterCleanup: engine.addCleanup, + }); err != nil { + return + } + if err := xhr.Enable(runtime); err != nil { + return + } return }) <-done diff --git a/pkg/download/engine/fetch_integration_test.go b/pkg/download/engine/fetch_integration_test.go new file mode 100644 index 00000000..dbe40afa --- /dev/null +++ b/pkg/download/engine/fetch_integration_test.go @@ -0,0 +1,168 @@ +package engine_test + +import ( + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/GopeedLab/gopeed/pkg/download/engine" +) + +func TestFetchDoesNotDependOnXMLHttpRequest(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/target", func(w http.ResponseWriter, request *http.Request) { + w.Header().Set("X-Method", request.Method) + _, _ = w.Write([]byte("ok")) + }) + mux.HandleFunc("/redirect", func(w http.ResponseWriter, request *http.Request) { + http.Redirect(w, request, "/target", http.StatusFound) + }) + server := httptest.NewServer(mux) + t.Cleanup(server.Close) + + runtime := engine.NewEngine(nil) + t.Cleanup(runtime.Close) + + value, err := runtime.RunString(fmt.Sprintf(` + (async () => { + globalThis.XMLHttpRequest = function () { + throw new Error("fetch used XMLHttpRequest"); + }; + + const get = await fetch(%q); + const head = await fetch(%q, { method: "HEAD" }); + const manual = await fetch(%q, { redirect: "manual" }); + let redirectError = false; + try { + await fetch(%q, { redirect: "error" }); + } catch (_) { + redirectError = true; + } + + return JSON.stringify({ + get: await get.text(), + headStatus: head.status, + headMethod: head.headers.get("x-method"), + manualStatus: manual.status, + manualLocation: manual.headers.get("location"), + redirectError, + }); + })() + `, server.URL+"/target", server.URL+"/target", server.URL+"/redirect", server.URL+"/redirect")) + if err != nil { + t.Fatal(err) + } + const want = `{"get":"ok","headStatus":200,"headMethod":"HEAD","manualStatus":302,"manualLocation":"/target","redirectError":true}` + if value != want { + t.Fatalf("unexpected fetch result: got %#v want %s", value, want) + } +} + +func TestXMLHttpRequestUsesFetch(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("X-Test", "value") + _, _ = w.Write([]byte("payload")) + })) + t.Cleanup(server.Close) + + runtime := engine.NewEngine(nil) + t.Cleanup(runtime.Close) + + value, err := runtime.RunString(fmt.Sprintf(` + (async () => { + const originalFetch = globalThis.fetch; + let fetchCalls = 0; + globalThis.fetch = function (...args) { + fetchCalls++; + return originalFetch(...args); + }; + + const result = await new Promise((resolve, reject) => { + const xhr = new XMLHttpRequest(); + const states = []; + xhr.onreadystatechange = () => states.push(xhr.readyState); + xhr.onerror = () => reject(new Error("XHR failed")); + xhr.onload = () => resolve({ + status: xhr.status, + text: xhr.responseText, + header: xhr.getResponseHeader("x-test"), + states, + }); + xhr.open("GET", %q); + xhr.send(); + }); + result.fetchCalls = fetchCalls; + return JSON.stringify(result); + })() + `, server.URL)) + if err != nil { + t.Fatal(err) + } + const want = `{"status":200,"text":"payload","header":"value","states":[1,2,3,4],"fetchCalls":1}` + if value != want { + t.Fatalf("unexpected XMLHttpRequest result: got %#v want %s", value, want) + } +} + +func TestFetchResponseAndRequestCloneHaveIndependentBodies(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + if request.Method == http.MethodPost { + _, _ = response.Write([]byte("posted")) + return + } + _, _ = response.Write([]byte("payload")) + })) + t.Cleanup(server.Close) + + runtime := engine.NewEngine(nil) + t.Cleanup(runtime.Close) + + value, err := runtime.RunString(fmt.Sprintf(` + (async () => { + const response = await fetch(%q); + const responseClone = response.clone(); + const responseBodies = await Promise.all([response.text(), responseClone.text()]); + + const request = new Request(%q, { method: "POST", body: "request-body" }); + const requestClone = request.clone(); + const requestBodies = await Promise.all([request.text(), requestClone.text()]); + + return JSON.stringify({ responseBodies, requestBodies }); + })() + `, server.URL, server.URL)) + if err != nil { + t.Fatal(err) + } + const want = `{"responseBodies":["payload","payload"],"requestBodies":["request-body","request-body"]}` + if value != want { + t.Fatalf("unexpected clone result: got %#v want %s", value, want) + } +} + +func TestFetchFormDataUsesWebEntryListSemantics(t *testing.T) { + runtime := engine.NewEngine(nil) + t.Cleanup(runtime.Close) + + value, err := runtime.RunString(` + (() => { + const data = new FormData(); + data.append("a", "one"); + data.append("a", 2); + data.append("b", "three"); + const beforeSet = Array.from(data.entries()); + data.set("a", "four"); + const afterSet = Array.from(data); + const all = data.getAll("a"); + data.delete("b"); + return JSON.stringify({ beforeSet, afterSet, all, hasB: data.has("b") }); + })() + `) + if err != nil { + t.Fatal(err) + } + const want = `{"beforeSet":[["a","one"],["a","2"],["b","three"]],"afterSet":[["a","four"],["b","three"]],"all":["four"],"hasB":false}` + if value != want { + t.Fatalf("unexpected FormData result: got %#v want %s", value, want) + } +} diff --git a/pkg/download/engine/inject/fetch/fetch.js b/pkg/download/engine/inject/fetch/fetch.js new file mode 100644 index 00000000..b49ef59f --- /dev/null +++ b/pkg/download/engine/inject/fetch/fetch.js @@ -0,0 +1,898 @@ +(function () { + const fetchOpen = globalThis.__gopeed_fetch_open; + const fetchRead = globalThis.__gopeed_fetch_read; + const fetchClose = globalThis.__gopeed_fetch_close; + const fetchAbort = globalThis.__gopeed_fetch_abort; + + class FormData { + constructor(form) { + if (form !== undefined) throw new TypeError("FormData HTML form construction is not available in an extension worker"); + this._entries = []; + } + + _normalizeValue(value) { + if (typeof Blob !== "undefined" && value instanceof Blob) return value; + if (value && typeof value === "object" && "name" in value && "size" in value) return value; + return String(value); + } + + append(name, value) { + this._entries.push([String(name), this._normalizeValue(value)]); + } + + delete(name) { + name = String(name); + this._entries = this._entries.filter((entry) => entry[0] !== name); + } + + get(name) { + name = String(name); + const entry = this._entries.find((item) => item[0] === name); + return entry ? entry[1] : null; + } + + getAll(name) { + name = String(name); + return this._entries.filter((entry) => entry[0] === name).map((entry) => entry[1]); + } + + has(name) { + name = String(name); + return this._entries.some((entry) => entry[0] === name); + } + + set(name, value) { + name = String(name); + value = this._normalizeValue(value); + const first = this._entries.findIndex((entry) => entry[0] === name); + if (first < 0) { + this._entries.push([name, value]); + return; + } + this._entries[first] = [name, value]; + this._entries = this._entries.filter((entry, index) => index === first || entry[0] !== name); + } + + _iterator(kind) { + const data = this; + let index = 0; + return { + next() { + if (index >= data._entries.length) return { value: undefined, done: true }; + const entry = data._entries[index++]; + if (kind === "key") return { value: entry[0], done: false }; + if (kind === "value") return { value: entry[1], done: false }; + return { value: [entry[0], entry[1]], done: false }; + }, + [Symbol.iterator]() { return this; }, + }; + } + + entries() { return this._iterator("entry"); } + keys() { return this._iterator("key"); } + values() { return this._iterator("value"); } + [Symbol.iterator]() { return this.entries(); } + + forEach(callback, thisArg) { + for (const [name, value] of this) callback.call(thisArg, value, name, this); + } + } + Object.defineProperty(FormData.prototype, Symbol.toStringTag, { configurable: true, value: "FormData" }); + globalThis.FormData = FormData; + + const headerNamePattern = /^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$/; + const headersIteratorBase = Object.getPrototypeOf(Object.getPrototypeOf([][Symbol.iterator]())); + const headersIteratorPrototype = Object.create(headersIteratorBase); + + function normalizeHeaderName(name) { + if (typeof name === "symbol") throw new TypeError("HTTP header names must be ByteStrings"); + name = String(name); + for (let index = 0; index < name.length; index++) { + if (name.charCodeAt(index) > 0xFF) throw new TypeError("HTTP header names must be ByteStrings"); + } + if (!headerNamePattern.test(name)) throw new TypeError("Invalid HTTP header name"); + return name.toLowerCase(); + } + + function normalizeHeaderValue(value) { + value = String(value); + for (let index = 0; index < value.length; index++) { + if (value.charCodeAt(index) > 0xFF) throw new TypeError("HTTP header values must be ByteStrings"); + } + value = value.replace(/^[\t\n\r ]+|[\t\n\r ]+$/g, ""); + if (/[\0\n\r]/.test(value)) throw new TypeError("Invalid HTTP header value"); + return value; + } + + function normalizeStatusText(value) { + value = String(value); + for (let index = 0; index < value.length; index++) { + const code = value.charCodeAt(index); + if (code > 0xFF || (code !== 0x09 && (code < 0x20 || code === 0x7F))) { + throw new TypeError("Invalid response status text"); + } + } + return value; + } + + function sortedHeaderEntries(headers) { + const result = []; + for (const name of Array.from(headers._values.keys()).sort()) { + const values = headers._values.get(name); + if (name === "set-cookie") { + for (const value of values) result.push([name, value]); + } else { + result.push([name, values.join(", ")]); + } + } + return result; + } + + Object.defineProperty(headersIteratorPrototype, "next", { + configurable: true, + enumerable: true, + writable: true, + value: function () { + const entries = sortedHeaderEntries(this._headers); + if (this._index >= entries.length) return { value: undefined, done: true }; + const entry = entries[this._index++]; + if (this._kind === "key") return { value: entry[0], done: false }; + if (this._kind === "value") return { value: entry[1], done: false }; + return { value: entry, done: false }; + }, + }); + Object.defineProperty(headersIteratorPrototype, Symbol.iterator, { + configurable: true, + enumerable: false, + writable: true, + value: function () { return this; }, + }); + + function createHeadersIterator(headers, kind) { + const iterator = Object.create(headersIteratorPrototype); + Object.defineProperties(iterator, { + _headers: { value: headers }, + _kind: { value: kind }, + _index: { value: 0, writable: true }, + }); + return iterator; + } + + function forbiddenRequestHeader(name, value) { + if (name.startsWith("proxy-") || name.startsWith("sec-")) return true; + if ([ + "accept-charset", "accept-encoding", "access-control-request-headers", + "access-control-request-method", "connection", "content-length", "cookie", + "cookie2", "date", "dnt", "expect", "host", "keep-alive", "origin", + "permissions-policy", "referer", "te", "trailer", "transfer-encoding", + "set-cookie", "upgrade", "via", + ].includes(name)) return true; + if (["x-http-method", "x-http-method-override", "x-method-override"].includes(name)) { + return String(value).split(",").some((method) => { + method = method.trim().toUpperCase(); + return method === "CONNECT" || method === "TRACE" || method === "TRACK"; + }); + } + return false; + } + + function noCORSSafelistedRequestHeader(name, value) { + if (value.length > 128) return false; + const hasUnsafeByte = /[\x00-\x08\x0A-\x1F\x7F"():<>?@\[\\\]{}]/.test(value); + if (name === "accept") return !hasUnsafeByte; + if (name === "accept-language" || name === "content-language") { + return /^[0-9A-Za-z *,\-.;=]*$/.test(value); + } + if (name === "content-type") { + if (hasUnsafeByte) return false; + const essence = value.split(";", 1)[0].trim().toLowerCase(); + return essence === "application/x-www-form-urlencoded" || + essence === "multipart/form-data" || essence === "text/plain"; + } + if (name === "range") return /^bytes=[0-9]+-[0-9]*$/.test(value); + return false; + } + + function blockedByGuard(headers, name, value) { + if (headers._guard === "immutable") throw new TypeError("Headers are immutable"); + if (headers._guard === "request") return forbiddenRequestHeader(name, value); + if (headers._guard === "request-no-cors") { + return forbiddenRequestHeader(name, value) || !noCORSSafelistedRequestHeader(name, value); + } + if (headers._guard === "response") return name === "set-cookie" || name === "set-cookie2"; + return false; + } + + class Headers { + constructor(init, guard) { + this._values = new Map(); + this._guard = guard || "none"; + if (init === undefined) return; + if (init === null || (typeof init !== "object" && typeof init !== "function")) { + throw new TypeError("Headers init must be a sequence or record"); + } + const iterator = init[Symbol.iterator]; + if (typeof iterator === "function") { + for (const pair of init) { + if (pair == null || typeof pair[Symbol.iterator] !== "function") { + throw new TypeError("Header pair must be iterable"); + } + const values = Array.from(pair); + if (values.length !== 2) throw new TypeError("Header pair must contain exactly two items"); + this.append(values[0], values[1]); + } + return; + } + for (const name of Reflect.ownKeys(init)) { + const descriptor = Reflect.getOwnPropertyDescriptor(init, name); + if (!descriptor || !descriptor.enumerable) continue; + const normalizedName = normalizeHeaderName(name); + this.append(normalizedName, init[name]); + } + } + + append(name, value) { + name = normalizeHeaderName(name); + value = normalizeHeaderValue(value); + const values = this._values.get(name); + const combinedValue = values && name !== "set-cookie" ? values.join(", ") + ", " + value : value; + if (blockedByGuard(this, name, combinedValue)) return; + if (values) values.push(value); + else this._values.set(name, [value]); + } + + delete(name) { + name = normalizeHeaderName(name); + if (this._guard === "immutable") throw new TypeError("Headers are immutable"); + if (this._guard === "request-no-cors") { + if (forbiddenRequestHeader(name, "") || !["accept", "accept-language", "content-language", "content-type", "range"].includes(name)) return; + this._values.delete(name); + return; + } + if (blockedByGuard(this, name, "")) return; + this._values.delete(name); + } + + get(name) { + const values = this._values.get(normalizeHeaderName(name)); + return values ? values.join(", ") : null; + } + + getSetCookie() { + const values = this._values.get("set-cookie"); + return values ? values.slice() : []; + } + + has(name) { + return this._values.has(normalizeHeaderName(name)); + } + + set(name, value) { + name = normalizeHeaderName(name); + value = normalizeHeaderValue(value); + if (blockedByGuard(this, name, value)) return; + this._values.set(name, [value]); + } + + keys() { return createHeadersIterator(this, "key"); } + values() { return createHeadersIterator(this, "value"); } + entries() { return createHeadersIterator(this, "entry"); } + [Symbol.iterator]() { return this.entries(); } + + forEach(callback, thisArg) { + for (const [name, value] of this) callback.call(thisArg, value, name, this); + } + } + + globalThis.Headers = Headers; + + const forbiddenMethods = new Set(["CONNECT", "TRACE", "TRACK"]); + + function normalizeMethod(method) { + method = String(method); + if (!headerNamePattern.test(method)) throw new TypeError("Invalid HTTP method"); + method = method.toUpperCase(); + if (forbiddenMethods.has(method)) throw new TypeError("Forbidden HTTP method"); + return method; + } + + function resolveURL(value) { + const base = globalThis.location && globalThis.location.href ? globalThis.location.href : "http://localhost/"; + const parsed = new URL(String(value), base); + if (parsed.username || parsed.password) throw new TypeError("Request URLs cannot contain credentials"); + return parsed.href; + } + + function normalizeEnum(value, allowed, name) { + value = String(value); + if (!allowed.includes(value)) throw new TypeError("Invalid Request " + name); + return value; + } + + function normalizeReferrer(value) { + value = String(value); + if (value === "") return "no-referrer"; + if (value === "about:client") return value; + return resolveURL(value); + } + + function copyArrayBuffer(value) { + if (value instanceof ArrayBuffer) return value.slice(0); + return value.buffer.slice(value.byteOffset, value.byteOffset + value.byteLength); + } + + function initializeBody(owner, body) { + owner._bodyInit = body == null ? null : body; + owner._bodyText = null; + owner._bodyArrayBuffer = null; + owner._bodyBlob = null; + owner._bodyFormData = null; + owner._bodyFormDataBoundary = ""; + owner._bodyStream = null; + owner._bodyUsed = false; + if (body == null) return; + if (typeof ReadableStream !== "undefined" && body instanceof ReadableStream) { + owner._bodyStream = body; + body.__gopeedMarkBodyUsed = () => { owner._bodyUsed = true; }; + } else if (typeof body === "string") { + owner._bodyText = body; + } else if (typeof Blob !== "undefined" && body instanceof Blob) { + owner._bodyBlob = body; + } else if (typeof FormData !== "undefined" && body instanceof FormData) { + owner._bodyFormData = body; + const suffix = crypto.randomUUID ? crypto.randomUUID() : String(Date.now()) + String(Math.random()).slice(2); + owner._bodyFormDataBoundary = "----gopeed-" + suffix; + } else if (body instanceof ArrayBuffer || (ArrayBuffer.isView && ArrayBuffer.isView(body))) { + owner._bodyArrayBuffer = copyArrayBuffer(body); + } else if (typeof URLSearchParams !== "undefined" && body instanceof URLSearchParams) { + owner._bodyText = body.toString(); + } else { + owner._bodyText = String(body); + } + } + + function bodyContentType(owner) { + if (owner._bodyText != null) { + if (typeof URLSearchParams !== "undefined" && owner._bodyInit instanceof URLSearchParams) { + return "application/x-www-form-urlencoded;charset=UTF-8"; + } + return "text/plain;charset=UTF-8"; + } + if (owner._bodyBlob && owner._bodyBlob.type) return owner._bodyBlob.type; + if (owner._bodyFormData) return "multipart/form-data; boundary=" + owner._bodyFormDataBoundary; + return ""; + } + + function decodeFormComponent(value) { + return decodeURIComponent(String(value).replace(/\+/g, " ")); + } + + function installBodyMethods(prototype) { + Object.defineProperties(prototype, { + body: { + configurable: true, + enumerable: true, + get() { + if (this._bodyInit == null && !this.__gopeedBodyStream) return null; + if (this.__gopeedBodyStream) return this.__gopeedBodyStream; + if (!this._bodyStream) this._bodyStream = createBodyReadableStream(this); + return this._bodyStream; + }, + }, + bodyUsed: { + configurable: true, + enumerable: true, + get() { return Boolean(this._bodyUsed || this.__gopeedBodyConsumed); }, + }, + }); + prototype.text = async function () { + if (this.bodyUsed) throw new TypeError("Already read"); + if (this.body == null) return ""; + return readAllFromStream(this.body, true); + }; + prototype.arrayBuffer = async function () { + if (this.bodyUsed) throw new TypeError("Already read"); + if (this.body == null) return new ArrayBuffer(0); + const bytes = await readAllFromStream(this.body, false); + return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength); + }; + prototype.bytes = async function () { + if (this.bodyUsed) throw new TypeError("Already read"); + if (this.body == null) return new Uint8Array(0); + return readAllFromStream(this.body, false); + }; + prototype.blob = async function () { + const data = await this.arrayBuffer(); + return new Blob([data], { type: this.headers.get("content-type") || "" }); + }; + prototype.json = async function () { return JSON.parse(await this.text()); }; + prototype.formData = async function () { + const contentType = this.headers.get("content-type") || ""; + const essence = contentType.split(";", 1)[0].trim().toLowerCase(); + if (essence === "multipart/form-data" && this._bodyFormData) { + if (this.bodyUsed) throw new TypeError("Already read"); + await readAllFromStream(this.body, false); + const result = new FormData(); + for (const entry of Array.from(this._bodyFormData.entries())) result.append(entry[0], entry[1]); + return result; + } + if (essence !== "application/x-www-form-urlencoded") { + throw new TypeError("Body is not URL-encoded form data"); + } + if (this.body == null) return new FormData(); + const text = await this.text(); + const result = new FormData(); + for (const entry of text.split("&")) { + if (!entry) continue; + const parts = entry.split("="); + const name = parts.shift(); + result.append(decodeFormComponent(name), decodeFormComponent(parts.join("="))); + } + return result; + }; + } + + function transferReadableStream(stream) { + let reader = null; + return new ReadableStream({ + async pull(controller) { + if (!reader) reader = stream.getReader(); + const result = await reader.read(); + if (result.done) { + reader.releaseLock(); + controller.close(); + } else { + controller.enqueue(result.value); + } + }, + cancel(reason) { + if (!reader) reader = stream.getReader(); + return reader.cancel(reason); + }, + }); + } + + class Request { + constructor(input, init) { + if (arguments.length < 1) throw new TypeError("Request requires an input"); + init = init || {}; + if (init.window !== undefined && init.window !== null) throw new TypeError("Request window must be null"); + const source = input instanceof Request ? input : null; + const bodyWasOverridden = Object.prototype.hasOwnProperty.call(init, "body"); + if (source && source.bodyUsed && !bodyWasOverridden) throw new TypeError("Request body is already used"); + this._url = source ? source.url : resolveURL(input); + this._method = normalizeMethod(init.method !== undefined ? init.method : (source ? source.method : "GET")); + this._destination = ""; + this._referrer = init.referrer !== undefined ? normalizeReferrer(init.referrer) : (source ? source.referrer : "about:client"); + this._referrerPolicy = normalizeEnum(init.referrerPolicy !== undefined ? init.referrerPolicy : (source ? source.referrerPolicy : ""), ["", "no-referrer", "no-referrer-when-downgrade", "origin", "origin-when-cross-origin", "same-origin", "strict-origin", "strict-origin-when-cross-origin", "unsafe-url"], "referrerPolicy"); + this._mode = normalizeEnum(init.mode !== undefined ? init.mode : (source ? source.mode : "cors"), ["same-origin", "no-cors", "cors"], "mode"); + this._credentials = normalizeEnum(init.credentials !== undefined ? init.credentials : (source ? source.credentials : "same-origin"), ["omit", "same-origin", "include"], "credentials"); + this._cache = normalizeEnum(init.cache !== undefined ? init.cache : (source ? source.cache : "default"), ["default", "no-store", "reload", "no-cache", "force-cache", "only-if-cached"], "cache"); + this._redirect = normalizeEnum(init.redirect !== undefined ? init.redirect : (source ? source.redirect : "follow"), ["follow", "error", "manual"], "redirect"); + if (this._mode === "no-cors" && !["GET", "HEAD", "POST"].includes(this._method)) { + throw new TypeError("no-cors requests require a CORS-safelisted method"); + } + if (this._cache === "only-if-cached" && this._mode !== "same-origin") { + throw new TypeError("only-if-cached requires same-origin mode"); + } + this._integrity = init.integrity !== undefined ? String(init.integrity) : (source ? source.integrity : ""); + this._keepalive = init.keepalive !== undefined ? Boolean(init.keepalive) : (source ? source.keepalive : false); + this._priority = normalizeEnum(init.priority !== undefined ? init.priority : (source ? source._priority : "auto"), ["high", "low", "auto"], "priority"); + this._signal = init.signal || (source && source.signal) || new AbortController().signal; + if (init.duplex !== undefined && String(init.duplex) !== "half") { + throw new TypeError("Request duplex must be 'half'"); + } + this._duplex = "half"; + const headerInit = init.headers !== undefined ? init.headers : (source ? source.headers : undefined); + this._headers = new Headers(headerInit, this._mode === "no-cors" ? "request-no-cors" : "request"); + let body = init.body !== undefined ? init.body : (source ? source._bodyInit : null); + if (source && !bodyWasOverridden && source.body) body = transferReadableStream(source.body); + const bodyIsStream = typeof ReadableStream !== "undefined" && body instanceof ReadableStream; + if (bodyIsStream && (body.locked || body._disturbed)) { + throw new TypeError("ReadableStream body is locked or disturbed"); + } + if (bodyIsStream && init.body !== undefined && init.duplex === undefined) { + throw new TypeError("ReadableStream request bodies require duplex: 'half'"); + } + if (bodyIsStream && this._keepalive) { + throw new TypeError("keepalive requests cannot have a ReadableStream body"); + } + if (source && !bodyWasOverridden && source.body && (source.body.locked || source.body._disturbed)) { + throw new TypeError("Request body is locked or disturbed"); + } + if ((this._method === "GET" || this._method === "HEAD") && body != null) { + throw new TypeError("Body not allowed for GET or HEAD requests"); + } + initializeBody(this, body); + const contentType = bodyContentType(this); + if (contentType && !this._headers.has("content-type")) this._headers.set("content-type", contentType); + if (source && source.body) source._bodyUsed = true; + } + + get method() { return this._method; } + get url() { return this._url; } + get headers() { return this._headers; } + get destination() { return this._destination; } + get referrer() { return this._referrer; } + get referrerPolicy() { return this._referrerPolicy; } + get mode() { return this._mode; } + get credentials() { return this._credentials; } + get cache() { return this._cache; } + get redirect() { return this._redirect; } + get integrity() { return this._integrity; } + get keepalive() { return this._keepalive; } + get signal() { return this._signal; } + get isReloadNavigation() { return false; } + get isHistoryNavigation() { return false; } + get duplex() { return this._duplex; } + clone() { + if (this.bodyUsed) throw new TypeError("Request body is already used"); + let body = null; + if (this.body) { + const branches = this.body.tee(); + this._bodyInit = branches[0]; + this._bodyStream = branches[0]; + if (this.__gopeedBodyStream) this.__gopeedBodyStream = branches[0]; + branches[0].__gopeedMarkBodyUsed = () => { this._bodyUsed = true; }; + body = branches[1]; + } + return new Request(this.url, { + method: this.method, + headers: this.headers, + body, + mode: this.mode, + credentials: this.credentials, + cache: this.cache, + redirect: this.redirect, + referrer: this.referrer, + referrerPolicy: this.referrerPolicy, + integrity: this.integrity, + keepalive: this.keepalive, + signal: this.signal, + duplex: "half", + }); + } + } + installBodyMethods(Request.prototype); + + const nullBodyStatuses = new Set([101, 103, 204, 205, 304]); + + class Response { + constructor(body, init) { + init = init || {}; + const status = init.status === undefined ? 200 : Number(init.status); + if (!Number.isInteger(status) || status < 200 || status > 599) { + throw new RangeError("Invalid response status"); + } + if (body != null && nullBodyStatuses.has(status)) { + throw new TypeError("Response with null-body status cannot have a body"); + } + const statusText = init.statusText === undefined ? "" : normalizeStatusText(init.statusText); + this._type = "default"; + this._url = init.url ? String(init.url) : ""; + this._redirected = Boolean(init.redirected); + this._status = status; + this._statusText = statusText; + this._headers = new Headers(init.headers, "response"); + if (typeof ReadableStream !== "undefined" && body instanceof ReadableStream && (body.locked || body._disturbed)) { + throw new TypeError("ReadableStream body is locked or disturbed"); + } + initializeBody(this, body == null ? null : body); + const contentType = bodyContentType(this); + if (contentType && !this._headers.has("content-type")) this._headers.set("content-type", contentType); + } + + get type() { return this._type; } + get url() { return this._url; } + get redirected() { return this._redirected; } + get status() { return this._status; } + get ok() { return this._status >= 200 && this._status <= 299; } + get statusText() { return this._statusText; } + get headers() { return this._headers; } + clone() { + if (this.bodyUsed) throw new TypeError("Response body is already used"); + let body = null; + if (this.body) { + const branches = this.body.tee(); + this._bodyInit = branches[0]; + this._bodyStream = branches[0]; + if (this.__gopeedBodyStream) this.__gopeedBodyStream = branches[0]; + branches[0].__gopeedMarkBodyUsed = () => { this._bodyUsed = true; }; + body = branches[1]; + } + return new Response(body, { + status: this.status, + statusText: this.statusText, + headers: this.headers, + url: this.url, + redirected: this.redirected, + }); + } + + static error() { + const response = Object.create(Response.prototype); + response._type = "error"; + response._url = ""; + response._redirected = false; + response._status = 0; + response._statusText = ""; + response._headers = new Headers(undefined, "immutable"); + initializeBody(response, null); + return response; + } + + static redirect(url, status) { + status = status === undefined ? 302 : Number(status); + if (![301, 302, 303, 307, 308].includes(status)) throw new RangeError("Invalid redirect status"); + return new Response(null, { status, headers: { location: resolveURL(url) } }); + } + + static json(data, init) { + const body = JSON.stringify(data); + if (body === undefined) throw new TypeError("Value is not JSON serializable"); + init = Object.assign({}, init || {}); + const headers = new Headers(init.headers); + if (!headers.has("content-type")) headers.set("content-type", "application/json"); + init.headers = headers; + return new Response(body, init); + } + } + installBodyMethods(Response.prototype); + + globalThis.Request = Request; + globalThis.Response = Response; + + function toUint8Array(chunk) { + if (chunk == null) return new Uint8Array(0); + if (chunk instanceof Uint8Array) return chunk; + if (typeof chunk === "string") return new TextEncoder().encode(chunk); + if (chunk instanceof ArrayBuffer) return new Uint8Array(chunk); + if (typeof ArrayBuffer !== "undefined" && ArrayBuffer.isView && ArrayBuffer.isView(chunk)) { + return new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength); + } + if (typeof Blob !== "undefined" && chunk instanceof Blob && chunk._buffer instanceof Uint8Array) { + return chunk._buffer; + } + return new Uint8Array(0); + } + + function createBodyReadableStream(owner) { + const stream = new ReadableStream({ + start(controller) { + Promise.resolve().then(async () => { + if (owner._noBody) { + controller.close(); + return; + } + if (owner._bodyArrayBuffer) { + controller.enqueue(toUint8Array(owner._bodyArrayBuffer)); + controller.close(); + return; + } + if (owner._bodyBlob) { + const data = await owner._bodyBlob.arrayBuffer(); + controller.enqueue(new Uint8Array(data)); + controller.close(); + return; + } + if (owner._bodyFormData) { + const boundary = owner._bodyFormDataBoundary; + const entries = Array.from(owner._bodyFormData.entries()); + if (entries.length === 0) { + controller.close(); + return; + } + let serialized = ""; + for (const entry of entries) { + const name = String(entry[0]).replace(/\r|\n|"/g, (character) => + character === "\r" ? "%0D" : character === "\n" ? "%0A" : "%22"); + serialized += "--" + boundary + "\r\nContent-Disposition: form-data; name=\"" + name + "\"\r\n\r\n"; + serialized += String(entry[1]) + "\r\n"; + } + serialized += "--" + boundary + "--\r\n"; + controller.enqueue(toUint8Array(serialized)); + controller.close(); + return; + } + if (owner._bodyText != null) { + controller.enqueue(toUint8Array(owner._bodyText)); + controller.close(); + return; + } + if (owner._bodyInit != null) controller.enqueue(toUint8Array(owner._bodyInit)); + controller.close(); + }).catch((error) => controller.error(error)); + }, + }); + stream.__gopeedMarkBodyUsed = () => { owner._bodyUsed = true; }; + return stream; + } + + async function readAllFromStream(stream, asText) { + if (!stream) return asText ? "" : new Uint8Array(0); + const reader = stream.getReader(); + const chunks = []; + let total = 0; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + if (!(value instanceof Uint8Array)) { + throw new TypeError("Fetch body stream chunks must be Uint8Array values"); + } + const chunk = value; + chunks.push(chunk); + total += chunk.byteLength; + } + } finally { + reader.releaseLock(); + } + const merged = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + merged.set(chunk, offset); + offset += chunk.byteLength; + } + return asText ? new TextDecoder().decode(merged) : merged; + } + + function attachResponseStreaming(response, stream) { + response.__gopeedBodyStream = stream; + response.__gopeedBodyConsumed = false; + const ensureUnused = () => { + if (response.__gopeedBodyConsumed) throw new TypeError("Already read"); + }; + const markBodyUsed = () => { + ensureUnused(); + response.__gopeedBodyConsumed = true; + response.bodyUsed = true; + }; + if (stream) { + stream.__gopeedMarkBodyUsed = () => { + if (!response.__gopeedBodyConsumed) { + response.__gopeedBodyConsumed = true; + response.bodyUsed = true; + } + }; + } + response.text = async function () { + markBodyUsed(); + return readAllFromStream(this.body, true); + }; + response.arrayBuffer = async function () { + markBodyUsed(); + const bytes = await readAllFromStream(this.body, false); + return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength); + }; + response.bytes = async function () { + markBodyUsed(); + return readAllFromStream(this.body, false); + }; + response.blob = async function () { + markBodyUsed(); + const contentType = this.headers && this.headers.get ? (this.headers.get("content-type") || "") : ""; + const bytes = await readAllFromStream(this.body, false); + return new Blob([bytes], { type: contentType }); + }; + response.json = async function () { + return JSON.parse(await this.text()); + }; + return response; + } + + if (typeof globalThis.Response === "function") { + const responseProto = globalThis.Response.prototype; + const bodyDescriptor = Object.getOwnPropertyDescriptor(responseProto, "body"); + if (!bodyDescriptor || typeof bodyDescriptor.get !== "function") { + Object.defineProperty(responseProto, "body", { + configurable: true, + enumerable: true, + get() { + if (!this.__gopeedBodyStream) this.__gopeedBodyStream = createBodyReadableStream(this); + return this.__gopeedBodyStream; + }, + }); + } + } + + if (typeof fetchOpen !== "function") return; + + globalThis.fetch = async function (input, init) { + const request = new Request(input, init); + const requestID = crypto.randomUUID ? crypto.randomUUID() : String(Date.now()) + "-" + String(Math.random()); + const signal = request.signal; + let abortHandler = null; + const cleanupAbort = () => { + if (signal && abortHandler && typeof signal.removeEventListener === "function") { + signal.removeEventListener("abort", abortHandler); + } + abortHandler = null; + }; + if (signal) { + if (signal.aborted) throw new DOMException("Aborted", "AbortError"); + abortHandler = () => fetchAbort(requestID, signal.reason == null ? "" : String(signal.reason)); + signal.addEventListener("abort", abortHandler, { once: true }); + } + + let body = null; + if (request._bodyStream && !request._bodyFormData) { + const bytes = await readAllFromStream(request._bodyStream, false); + body = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength); + } else if (request._bodyFormData) { + body = request._bodyFormData._entries; + } else if (request._bodyArrayBuffer) { + body = request._bodyArrayBuffer; + } else if (request._bodyBlob) { + body = await request._bodyBlob.arrayBuffer(); + } else if (request._bodyInit != null && typeof request._bodyInit === "object") { + body = request._bodyInit; + } else if (request._bodyText != null) { + body = request._bodyText; + } else if (request._bodyInit != null) { + body = request._bodyInit; + } + + const headers = []; + request.headers.forEach((value, key) => headers.push([key, value])); + let metadata; + try { + metadata = await fetchOpen({ + id: requestID, + url: request.url, + method: request.method, + headers, + body, + bodyType: request._bodyFormData ? "formdata" : "", + bodyBoundary: request._bodyFormDataBoundary, + redirect: request.redirect, + credentials: request.credentials, + }); + } catch (error) { + cleanupAbort(); + if (signal && signal.aborted) throw new DOMException("Aborted", "AbortError"); + throw error instanceof Error ? error : new TypeError(String(error)); + } + + const stream = new ReadableStream({ + async pull(controller) { + let chunk; + try { + chunk = await fetchRead(metadata.id, 64 * 1024); + } catch (error) { + fetchClose(metadata.id); + cleanupAbort(); + controller.error(error instanceof Error ? error : new TypeError(String(error))); + return; + } + if (chunk == null) { + fetchClose(metadata.id); + cleanupAbort(); + controller.close(); + return; + } + const bytes = chunk instanceof Uint8Array ? chunk : new Uint8Array(chunk); + if (bytes.byteLength === 0) { + fetchClose(metadata.id); + cleanupAbort(); + controller.close(); + return; + } + controller.enqueue(bytes); + }, + cancel(reason) { + cleanupAbort(); + fetchAbort(metadata.id, reason == null ? "" : String(reason)); + }, + }); + const response = new Response(null, { + status: metadata.status, + statusText: metadata.statusText, + headers: metadata.headers, + url: metadata.url, + redirected: metadata.redirected, + }); + response._type = "basic"; + response._headers._guard = "immutable"; + return attachResponseStreaming(response, stream); + }; +})(); diff --git a/pkg/download/engine/inject/fetch/module.go b/pkg/download/engine/inject/fetch/module.go new file mode 100644 index 00000000..f882fd7c --- /dev/null +++ b/pkg/download/engine/inject/fetch/module.go @@ -0,0 +1,619 @@ +package fetch + +import ( + "bytes" + "context" + _ "embed" + "errors" + "fmt" + "io" + "mime/multipart" + "net" + "net/http" + "net/http/cookiejar" + "net/url" + "strings" + "sync" + + "github.com/GopeedLab/gopeed/internal/httpclient" + "github.com/GopeedLab/gopeed/pkg/download/engine/inject/file" + "github.com/GopeedLab/gopeed/pkg/download/engine/util" + "github.com/dop251/goja" + "github.com/dop251/goja_nodejs/eventloop" + "golang.org/x/net/publicsuffix" +) + +const FingerprintMagicKey = "__gopeed_fetch_fingerprint" + +//go:embed fetch.js +var script string + +type Config struct { + ProxyHandler func(*http.Request) (*url.URL, error) + RegisterCleanup func(func()) +} + +func Enable(runtime *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error { + if cfg == nil { + cfg = &Config{} + } + registry := newRegistry() + if cfg.RegisterCleanup != nil { + cfg.RegisterCleanup(registry.CloseAll) + } + if err := runtime.Set("__gopeed_setFingerprint", func(fingerprint string) { + _ = runtime.Set(FingerprintMagicKey, fingerprint) + }); err != nil { + return err + } + if err := runtime.Set("__gopeed_fetch_open", func(call goja.FunctionCall) goja.Value { + request, err := exportRequest(runtime, call.Argument(0)) + if err != nil { + panic(runtime.NewGoError(err)) + } + fingerprint := util.SafeGet[string](runtime, FingerprintMagicKey) + promise, resolve, reject := runtime.NewPromise() + go func() { + metadata, err := registry.Open(fingerprint, cfg.ProxyHandler, request) + ok := loop.RunOnLoop(func(runtime *goja.Runtime) { + if err != nil { + reject(runtime.NewGoError(err)) + return + } + resolve(runtime.ToValue(metadata)) + }) + if !ok && metadata != nil { + registry.Close(metadata.ID) + } + }() + return runtime.ToValue(promise) + }); err != nil { + return err + } + if err := runtime.Set("__gopeed_fetch_read", func(call goja.FunctionCall) goja.Value { + id := call.Argument(0).String() + chunkSize := int(call.Argument(1).ToInteger()) + promise, resolve, reject := runtime.NewPromise() + go func() { + chunk, done, err := registry.Read(id, chunkSize) + ok := loop.RunOnLoop(func(runtime *goja.Runtime) { + if err != nil { + reject(runtime.NewGoError(err)) + return + } + if done { + resolve(goja.Null()) + return + } + resolve(runtime.ToValue(runtime.NewArrayBuffer(chunk))) + }) + if !ok { + registry.Close(id) + } + }() + return runtime.ToValue(promise) + }); err != nil { + return err + } + if err := runtime.Set("__gopeed_fetch_close", func(id string) { + registry.Close(id) + }); err != nil { + return err + } + if err := runtime.Set("__gopeed_fetch_abort", func(id, reason string) { + registry.Abort(id, reason) + }); err != nil { + return err + } + _, err := runtime.RunString(script) + return err +} + +type request struct { + ID string + URL string + Method string + Headers [][2]string + Body any + Redirect string + Credentials string +} + +type responseMetadata struct { + ID string `json:"id"` + Status int `json:"status"` + StatusText string `json:"statusText"` + URL string `json:"url"` + Redirected bool `json:"redirected"` + Headers [][2]string `json:"headers"` +} + +type formDataEntry struct { + name string + value any +} + +type formDataSnapshot struct { + entries []formDataEntry + boundary string +} + +func exportRequest(runtime *goja.Runtime, value goja.Value) (*request, error) { + obj := value.ToObject(runtime) + if obj == nil { + return nil, fmt.Errorf("invalid fetch request") + } + result := &request{ + ID: obj.Get("id").String(), + URL: obj.Get("url").String(), + Method: obj.Get("method").String(), + Redirect: obj.Get("redirect").String(), + Credentials: obj.Get("credentials").String(), + } + if result.Method == "" { + result.Method = http.MethodGet + } + if headersValue := obj.Get("headers"); headersValue != nil && !goja.IsUndefined(headersValue) && !goja.IsNull(headersValue) { + if exported, ok := headersValue.Export().([]any); ok { + for _, item := range exported { + pair, ok := item.([]any) + if !ok || len(pair) != 2 { + continue + } + result.Headers = append(result.Headers, [2]string{fmt.Sprint(pair[0]), fmt.Sprint(pair[1])}) + } + } + } + bodyValue := obj.Get("body") + if bodyValue != nil && !goja.IsUndefined(bodyValue) && !goja.IsNull(bodyValue) { + var body any + var err error + if obj.Get("bodyType").String() == "formdata" { + body, err = snapshotFormData(bodyValue.Export()) + } else { + body, err = snapshotBody(bodyValue.Export()) + } + if err != nil { + return nil, err + } + if snapshot, ok := body.(*formDataSnapshot); ok { + snapshot.boundary = obj.Get("bodyBoundary").String() + } + result.Body = body + } + return result, nil +} + +// snapshotBody runs on the JavaScript event loop. Go request goroutines must +// not retain mutable ArrayBuffer or FormData storage owned by goja. +func snapshotBody(body any) (any, error) { + switch value := body.(type) { + case nil, string: + return value, nil + case []byte: + return append([]byte(nil), value...), nil + case goja.ArrayBuffer: + return append([]byte(nil), value.Bytes()...), nil + case *file.File: + return snapshotFile(value), nil + default: + if bytesValue, ok := value.(interface{ Bytes() []byte }); ok { + return append([]byte(nil), bytesValue.Bytes()...), nil + } + return value, nil + } +} + +func snapshotFormData(body any) (*formDataSnapshot, error) { + entries, ok := body.([]any) + if !ok { + return nil, fmt.Errorf("invalid FormData entry list %T", body) + } + snapshot := &formDataSnapshot{entries: make([]formDataEntry, 0, len(entries))} + for _, item := range entries { + pair, ok := item.([]any) + if !ok || len(pair) != 2 { + return nil, fmt.Errorf("invalid FormData entry %T", item) + } + name, ok := pair[0].(string) + if !ok { + return nil, fmt.Errorf("invalid FormData field name %T", pair[0]) + } + value := pair[1] + if formFile, ok := value.(*file.File); ok { + value = snapshotFile(formFile) + } + snapshot.entries = append(snapshot.entries, formDataEntry{name: name, value: value}) + } + return snapshot, nil +} + +func snapshotFile(source *file.File) *file.File { + if source == nil { + return nil + } + return &file.File{ + Reader: source.Reader, + Closer: source.Closer, + Name: source.Name, + Size: source.Size, + } +} + +type registry struct { + mu sync.Mutex + operations map[string]*operation + ctx context.Context + cancel context.CancelFunc + jar http.CookieJar + closed bool +} + +type operation struct { + ctx context.Context + cancel context.CancelFunc + readMu sync.Mutex + stateMu sync.Mutex + body io.ReadCloser + pending error + closed bool + closeOnce sync.Once +} + +func newRegistry() *registry { + ctx, cancel := context.WithCancel(context.Background()) + jar, _ := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List}) + return ®istry{ + operations: make(map[string]*operation), + ctx: ctx, + cancel: cancel, + jar: jar, + } +} + +func (r *registry) Open(fingerprint string, proxyHandler func(*http.Request) (*url.URL, error), metadata *request) (*responseMetadata, error) { + ctx, cancel := context.WithCancel(r.ctx) + op := &operation{ctx: ctx, cancel: cancel} + if err := r.add(metadata.ID, op); err != nil { + cancel() + return nil, err + } + keepOperation := false + defer func() { + if !keepOperation { + r.Close(metadata.ID) + } + }() + + contentType, body, contentLength, err := buildBody(metadata.Body) + if err != nil { + return nil, err + } + if metadata.Method == http.MethodGet || metadata.Method == http.MethodHead { + body = nil + contentLength = 0 + } + httpRequest, err := http.NewRequestWithContext(ctx, metadata.Method, metadata.URL, body) + if err != nil { + return nil, err + } + for _, header := range metadata.Headers { + httpRequest.Header.Add(header[0], header[1]) + } + if host := httpRequest.Header.Get("Host"); host != "" { + httpRequest.Host = host + } + if body != nil { + if contentType != "" && !hasHeader(metadata.Headers, "Content-Type") { + httpRequest.Header.Set("Content-Type", contentType) + } + httpRequest.ContentLength = contentLength + } + + var jar http.CookieJar + if metadata.Credentials != "omit" { + jar = r.jar + } + redirected := false + client, err := httpclient.NewClient(httpclient.Options{ + Client: httpclient.ClientOptions{ + Jar: jar, + CheckRedirect: func(_ *http.Request, via []*http.Request) error { + switch metadata.Redirect { + case "manual": + return http.ErrUseLastResponse + case "error": + return fmt.Errorf("redirect failed") + default: + redirected = len(via) > 0 + if len(via) > 20 { + return fmt.Errorf("too many redirects") + } + return nil + } + }, + }, + Transport: httpclient.TransportOptions{Proxy: proxyHandler}, + Impersonation: impersonationOptions(fingerprint), + }) + if err != nil { + return nil, err + } + response, err := client.Do(httpRequest) + if err != nil { + var networkError net.Error + if errors.As(err, &networkError) && networkError.Timeout() { + return nil, fmt.Errorf("Network request timed out") + } + return nil, fmt.Errorf("Network request failed: %w", err) + } + if !op.setBody(response.Body) { + return nil, context.Canceled + } + keepOperation = true + result := &responseMetadata{ + ID: metadata.ID, + Status: response.StatusCode, + StatusText: http.StatusText(response.StatusCode), + URL: metadata.URL, + } + if response.Request != nil && response.Request.URL != nil { + responseURL := *response.Request.URL + responseURL.Fragment = "" + result.URL = responseURL.String() + } + result.Redirected = redirected + for key, values := range response.Header { + for _, value := range values { + result.Headers = append(result.Headers, [2]string{key, value}) + } + } + return result, nil +} + +func (r *registry) add(id string, op *operation) error { + if id == "" { + return fmt.Errorf("fetch request ID is required") + } + r.mu.Lock() + defer r.mu.Unlock() + if r.closed { + return context.Canceled + } + if _, exists := r.operations[id]; exists { + return fmt.Errorf("duplicate fetch request ID %q", id) + } + r.operations[id] = op + return nil +} + +func (r *registry) Read(id string, chunkSize int) ([]byte, bool, error) { + op := r.get(id) + if op == nil { + return nil, true, nil + } + chunk, done, err := op.read(chunkSize) + if done || err != nil { + r.Close(id) + } + return chunk, done, err +} + +func (r *registry) Close(id string) { + r.mu.Lock() + op := r.operations[id] + delete(r.operations, id) + r.mu.Unlock() + if op != nil { + op.close() + } +} + +func (r *registry) Abort(id, _ string) { + r.Close(id) +} + +func (r *registry) CloseAll() { + r.mu.Lock() + if r.closed { + r.mu.Unlock() + return + } + r.closed = true + operations := r.operations + r.operations = make(map[string]*operation) + r.mu.Unlock() + r.cancel() + for _, op := range operations { + op.close() + } +} + +func (r *registry) get(id string) *operation { + r.mu.Lock() + defer r.mu.Unlock() + return r.operations[id] +} + +func (o *operation) setBody(body io.ReadCloser) bool { + o.stateMu.Lock() + defer o.stateMu.Unlock() + if o.closed { + _ = body.Close() + return false + } + o.body = body + return true +} + +func (o *operation) read(chunkSize int) ([]byte, bool, error) { + o.readMu.Lock() + defer o.readMu.Unlock() + o.stateMu.Lock() + body := o.body + pending := o.pending + o.pending = nil + closed := o.closed + o.stateMu.Unlock() + if pending != nil { + if errors.Is(pending, io.EOF) { + return nil, true, nil + } + return nil, false, pending + } + if closed || body == nil { + return nil, true, nil + } + if chunkSize <= 0 { + chunkSize = 64 * 1024 + } + buffer := make([]byte, chunkSize) + n, err := body.Read(buffer) + if n > 0 { + if err != nil { + o.stateMu.Lock() + o.pending = err + o.stateMu.Unlock() + } + return buffer[:n], false, nil + } + if errors.Is(err, io.EOF) { + return nil, true, nil + } + if err != nil { + return nil, false, err + } + return []byte{}, false, nil +} + +func (o *operation) close() { + o.closeOnce.Do(func() { + o.cancel() + o.stateMu.Lock() + o.closed = true + body := o.body + o.stateMu.Unlock() + if body != nil { + _ = body.Close() + } + }) +} + +func buildBody(body any) (string, io.Reader, int64, error) { + switch value := body.(type) { + case nil: + return "", nil, 0, nil + case string: + return "text/plain;charset=UTF-8", strings.NewReader(value), int64(len(value)), nil + case []byte: + return "", bytes.NewReader(value), int64(len(value)), nil + case goja.ArrayBuffer: + data := value.Bytes() + return "", bytes.NewReader(data), int64(len(data)), nil + case *file.File: + return "application/octet-stream", value.Reader, value.Size, nil + case *formDataSnapshot: + return buildMultipartBody(value) + default: + if bytesValue, ok := value.(interface{ Bytes() []byte }); ok { + data := bytesValue.Bytes() + return "", bytes.NewReader(data), int64(len(data)), nil + } + data := fmt.Sprint(value) + return "", strings.NewReader(data), int64(len(data)), nil + } +} + +type countingWriter struct { + n int64 +} + +func (w *countingWriter) Write(data []byte) (int, error) { + w.n += int64(len(data)) + return len(data), nil +} + +func buildMultipartBody(snapshot *formDataSnapshot) (string, io.Reader, int64, error) { + boundary := snapshot.boundary + if boundary == "" { + boundaryWriter := multipart.NewWriter(io.Discard) + boundary = boundaryWriter.Boundary() + _ = boundaryWriter.Close() + } + counter := &countingWriter{} + statWriter := multipart.NewWriter(counter) + if err := statWriter.SetBoundary(boundary); err != nil { + return "", nil, 0, err + } + for _, entry := range snapshot.entries { + switch value := entry.value.(type) { + case string: + if err := statWriter.WriteField(entry.name, value); err != nil { + return "", nil, 0, err + } + case *file.File: + if _, err := statWriter.CreateFormFile(entry.name, value.Name); err != nil { + return "", nil, 0, err + } + counter.n += value.Size + } + } + if err := statWriter.Close(); err != nil { + return "", nil, 0, err + } + + reader, writer := io.Pipe() + multipartWriter := multipart.NewWriter(writer) + if err := multipartWriter.SetBoundary(boundary); err != nil { + _ = reader.Close() + _ = writer.CloseWithError(err) + return "", nil, 0, err + } + go func() { + for _, entry := range snapshot.entries { + var err error + switch value := entry.value.(type) { + case string: + err = multipartWriter.WriteField(entry.name, value) + case *file.File: + var part io.Writer + part, err = multipartWriter.CreateFormFile(entry.name, value.Name) + if err == nil { + _, err = io.Copy(part, value) + } + } + if err != nil { + _ = writer.CloseWithError(err) + return + } + } + if err := multipartWriter.Close(); err != nil { + _ = writer.CloseWithError(err) + return + } + _ = writer.Close() + }() + return multipartWriter.FormDataContentType(), reader, counter.n, nil +} + +func hasHeader(headers [][2]string, key string) bool { + for _, header := range headers { + if strings.EqualFold(header[0], key) { + return true + } + } + return false +} + +func impersonationOptions(fingerprint string) httpclient.ImpersonationOptions { + switch fingerprint { + case "chrome": + return httpclient.ImpersonationOptions{Mode: httpclient.ImpersonationFixed, Browser: httpclient.BrowserChrome} + case "firefox": + return httpclient.ImpersonationOptions{Mode: httpclient.ImpersonationFixed, Browser: httpclient.BrowserFirefox} + case "safari": + return httpclient.ImpersonationOptions{Mode: httpclient.ImpersonationFixed, Browser: httpclient.BrowserSafari} + default: + return httpclient.ImpersonationOptions{Mode: httpclient.ImpersonationNative} + } +} diff --git a/pkg/download/engine/inject/formdata/module.go b/pkg/download/engine/inject/formdata/module.go deleted file mode 100644 index 7c5388e7..00000000 --- a/pkg/download/engine/inject/formdata/module.go +++ /dev/null @@ -1,68 +0,0 @@ -package formdata - -import "github.com/dop251/goja" - -type FormData struct { - data map[string]any -} - -func (fd *FormData) Append(name string, value any) { - fd.data[name] = value -} - -func (fd *FormData) Delete(name string) { - delete(fd.data, name) -} - -func (fd *FormData) Entries() []any { - var entries []any - for k, v := range fd.data { - entries = append(entries, []any{k, v}) - } - return entries -} - -func (fd *FormData) Get(name string) any { - return fd.data[name] -} - -func (fd *FormData) GetAll(name string) []any { - return []any{fd.data[name]} -} - -func (fd *FormData) Has(name string) bool { - _, ok := fd.data[name] - return ok -} - -func (fd *FormData) Keys() []string { - var keys []string - for k := range fd.data { - keys = append(keys, k) - } - return keys -} - -func (fd *FormData) Set(name string, value any) { - fd.data[name] = value -} - -func (fd *FormData) Values() []any { - var values []any - for _, v := range fd.data { - values = append(values, v) - } - return values -} - -func Enable(runtime *goja.Runtime) error { - file := runtime.ToValue(func(call goja.ConstructorCall) *goja.Object { - instance := &FormData{ - data: make(map[string]any), - } - instanceValue := runtime.ToValue(instance).(*goja.Object) - instanceValue.SetPrototype(call.This.Prototype()) - return instanceValue - }) - return runtime.Set("FormData", file) -} diff --git a/pkg/download/engine/inject/stream/module.go b/pkg/download/engine/inject/stream/module.go index 65af2d6b..5aaa214f 100644 --- a/pkg/download/engine/inject/stream/module.go +++ b/pkg/download/engine/inject/stream/module.go @@ -6,21 +6,11 @@ import ( "errors" "fmt" "io" - "net" - "net/http" - "net/url" - "strings" "sync" "sync/atomic" - "time" - "github.com/GopeedLab/gopeed/pkg/download/engine/inject/file" - "github.com/GopeedLab/gopeed/pkg/download/engine/inject/formdata" - "github.com/GopeedLab/gopeed/pkg/download/engine/inject/xhr" - "github.com/GopeedLab/gopeed/pkg/download/engine/util" "github.com/dop251/goja" "github.com/dop251/goja_nodejs/eventloop" - "github.com/imroc/req/v3" ) //go:embed stream.js @@ -29,8 +19,6 @@ var script string type Config struct { CreateObjectURL func(opts *ObjectURLOptions, open ObjectURLOpener) (string, error) RevokeObjectURL func(url string) error - ProxyHandler func(r *http.Request) (*url.URL, error) - RegisterCleanup func(cleanup func()) } type ObjectURLOptions struct { @@ -96,69 +84,6 @@ func Enable(runtime *goja.Runtime, loop *eventloop.EventLoop, cfg *Config) error }); err != nil { return err } - fetchRegistry := newFetchRegistry() - if cfg.RegisterCleanup != nil { - cfg.RegisterCleanup(fetchRegistry.CloseAll) - } - if err := runtime.Set("__gopeed_fetch_open", func(call goja.FunctionCall) goja.Value { - reqMeta, err := exportFetchRequest(runtime, call.Argument(0)) - if err != nil { - panic(runtime.NewGoError(err)) - } - fingerprint := util.SafeGet[string](runtime, xhr.FingerprintMagicKey) - promise, resolve, reject := runtime.NewPromise() - go func() { - meta, err := fetchRegistry.Open(fingerprint, cfg.ProxyHandler, reqMeta) - ok := loop.RunOnLoop(func(runtime *goja.Runtime) { - if err != nil { - reject(runtime.NewGoError(err)) - return - } - resolve(runtime.ToValue(meta)) - }) - if !ok && meta != nil { - fetchRegistry.Close(meta.ID) - } - }() - return runtime.ToValue(promise) - }); err != nil { - return err - } - if err := runtime.Set("__gopeed_fetch_read", func(call goja.FunctionCall) goja.Value { - id := call.Argument(0).String() - chunkSize := int(call.Argument(1).ToInteger()) - promise, resolve, reject := runtime.NewPromise() - go func() { - chunk, done, err := fetchRegistry.Read(id, chunkSize) - ok := loop.RunOnLoop(func(runtime *goja.Runtime) { - if err != nil { - reject(runtime.NewGoError(err)) - return - } - if done { - resolve(goja.Null()) - return - } - resolve(runtime.ToValue(runtime.NewArrayBuffer(chunk))) - }) - if !ok { - fetchRegistry.Close(id) - } - }() - return runtime.ToValue(promise) - }); err != nil { - return err - } - if err := runtime.Set("__gopeed_fetch_close", func(id string) { - fetchRegistry.Close(id) - }); err != nil { - return err - } - if err := runtime.Set("__gopeed_fetch_abort", func(id string, reason string) { - fetchRegistry.Abort(id, reason) - }); err != nil { - return err - } if err := runtime.Set("__gopeed_blob_pipe_chunk", func(call goja.FunctionCall) goja.Value { id := call.Argument(0).String() ok := blobPipes.Push(id, exportBytes(call.Argument(1))) @@ -550,441 +475,3 @@ func exportJSError(value goja.Value) error { } return errors.New(stack) } - -type fetchRegistry struct { - mu sync.Mutex - streams map[string]*fetchStream - ctx context.Context - cancel context.CancelFunc - closed bool -} - -type fetchStream struct { - body io.ReadCloser - cancel context.CancelFunc - closeOnce sync.Once - ctx context.Context - ch chan fetchChunk - readMu sync.Mutex - pending []byte -} - -type fetchChunk struct { - data []byte - err error -} - -type fetchRequest struct { - URL string - Method string - Headers [][2]string - Body any - Redirect string - Credentials string -} - -type fetchFormDataEntry struct { - name string - value any -} - -type fetchFormDataSnapshot struct { - entries []fetchFormDataEntry -} - -type fetchOpenMeta struct { - ID string `json:"id"` - Status int `json:"status"` - StatusText string `json:"statusText"` - URL string `json:"url"` - Headers [][2]string `json:"headers"` -} - -func newFetchRegistry() *fetchRegistry { - ctx, cancel := context.WithCancel(context.Background()) - return &fetchRegistry{ - streams: make(map[string]*fetchStream), - ctx: ctx, - cancel: cancel, - } -} - -func exportFetchRequest(runtime *goja.Runtime, value goja.Value) (*fetchRequest, error) { - obj := value.ToObject(runtime) - if obj == nil { - return nil, fmt.Errorf("invalid fetch request") - } - meta := &fetchRequest{ - URL: obj.Get("url").String(), - Method: obj.Get("method").String(), - Redirect: obj.Get("redirect").String(), - Credentials: obj.Get("credentials").String(), - } - if meta.Method == "" { - meta.Method = http.MethodGet - } - if headersVal := obj.Get("headers"); headersVal != nil && !goja.IsUndefined(headersVal) && !goja.IsNull(headersVal) { - if exported, ok := headersVal.Export().([]any); ok { - for _, item := range exported { - pair, ok := item.([]any) - if !ok || len(pair) != 2 { - continue - } - meta.Headers = append(meta.Headers, [2]string{fmt.Sprint(pair[0]), fmt.Sprint(pair[1])}) - } - } - } - bodyVal := obj.Get("body") - if bodyVal != nil && !goja.IsUndefined(bodyVal) && !goja.IsNull(bodyVal) { - body, err := snapshotFetchBody(bodyVal.Export()) - if err != nil { - return nil, err - } - meta.Body = body - } - return meta, nil -} - -// snapshotFetchBody runs on the JavaScript event loop before fetch work is -// handed to a background goroutine. Values exported by goja may still share -// mutable storage with JavaScript (notably ArrayBuffer and FormData), so the -// background request must only retain immutable Go-owned data. -func snapshotFetchBody(body any) (any, error) { - switch v := body.(type) { - case nil, string: - return v, nil - case []byte: - return append([]byte(nil), v...), nil - case goja.ArrayBuffer: - return append([]byte(nil), v.Bytes()...), nil - case *file.File: - return snapshotFetchFile(v), nil - case *formdata.FormData: - entries := v.Entries() - snapshot := &fetchFormDataSnapshot{ - entries: make([]fetchFormDataEntry, 0, len(entries)), - } - for _, entry := range entries { - pair, ok := entry.([]any) - if !ok || len(pair) != 2 { - return nil, fmt.Errorf("invalid FormData entry %T", entry) - } - name, ok := pair[0].(string) - if !ok { - return nil, fmt.Errorf("invalid FormData field name %T", pair[0]) - } - value := pair[1] - if formFile, ok := value.(*file.File); ok { - value = snapshotFetchFile(formFile) - } - snapshot.entries = append(snapshot.entries, fetchFormDataEntry{ - name: name, - value: value, - }) - } - return snapshot, nil - default: - if typed, ok := v.(interface{ Bytes() []byte }); ok { - return append([]byte(nil), typed.Bytes()...), nil - } - return v, nil - } -} - -func snapshotFetchFile(src *file.File) *file.File { - if src == nil { - return nil - } - return &file.File{ - Reader: src.Reader, - Closer: src.Closer, - Name: src.Name, - Size: src.Size, - } -} - -func (r *fetchRegistry) Open(fingerprint string, proxyHandler func(r *http.Request) (*url.URL, error), reqMeta *fetchRequest) (*fetchOpenMeta, error) { - r.mu.Lock() - if r.closed { - r.mu.Unlock() - return nil, context.Canceled - } - registryCtx := r.ctx - r.mu.Unlock() - - client := req.C() - if proxyHandler != nil { - client.SetProxy(proxyHandler) - } - setFetchFingerprint(client, fingerprint) - contentType, body, err := buildFetchBody(reqMeta.Body) - if err != nil { - return nil, err - } - ctx, cancel := context.WithCancel(registryCtx) - keepCancel := false - defer func() { - if !keepCancel { - cancel() - } - }() - reqBuilder := client.R() - reqBuilder.SetContext(ctx) - reqBuilder.DisableAutoReadResponse() - for _, header := range reqMeta.Headers { - reqBuilder.SetHeader(header[0], header[1]) - } - if body != nil && reqMeta.Method != http.MethodGet && reqMeta.Method != http.MethodHead { - reqBuilder.SetBody(body) - if contentType != "" && !hasHeader(reqMeta.Headers, "Content-Type") { - reqBuilder.SetHeader("Content-Type", contentType) - } - } - client.SetRedirectPolicy(func(req *http.Request, via []*http.Request) error { - switch reqMeta.Redirect { - case "manual": - return http.ErrUseLastResponse - case "error": - return fmt.Errorf("redirect failed") - default: - if len(via) > 20 { - return fmt.Errorf("too many redirects") - } - return nil - } - }) - resp, err := reqBuilder.Send(reqMeta.Method, reqMeta.URL) - if err != nil { - var ne net.Error - if errorsAsTimeout(err, &ne) { - return nil, fmt.Errorf("Network request timed out") - } - return nil, fmt.Errorf("Network request failed: %w", err) - } - id := fmt.Sprintf("%d", time.Now().UnixNano()) - meta := &fetchOpenMeta{ - ID: id, - Status: resp.StatusCode, - StatusText: resp.Status, - URL: reqMeta.URL, - } - if resp.Response != nil && resp.Response.Request != nil && resp.Response.Request.URL != nil { - responseURL := *resp.Response.Request.URL - responseURL.Fragment = "" - meta.URL = responseURL.String() - } - for key, values := range resp.Header { - meta.Headers = append(meta.Headers, [2]string{key, strings.Join(values, ", ")}) - } - bodyCloser := io.NopCloser(strings.NewReader("")) - if resp.Response != nil && resp.Response.Body != nil { - bodyCloser = resp.Response.Body - } - stream := &fetchStream{ - body: bodyCloser, - cancel: cancel, - ctx: ctx, - ch: make(chan fetchChunk, 8), - } - r.mu.Lock() - if r.closed { - r.mu.Unlock() - closeFetchStream(stream) - return nil, context.Canceled - } - r.streams[id] = stream - r.mu.Unlock() - keepCancel = true - go stream.readLoop() - return meta, nil -} - -func (r *fetchRegistry) Read(id string, chunkSize int) ([]byte, bool, error) { - stream := r.get(id) - if stream == nil { - return nil, true, nil - } - stream.readMu.Lock() - defer stream.readMu.Unlock() - if len(stream.pending) > 0 { - n := len(stream.pending) - if chunkSize > 0 && n > chunkSize { - n = chunkSize - } - chunk := append([]byte(nil), stream.pending[:n]...) - stream.pending = stream.pending[n:] - return chunk, false, nil - } - select { - case item, ok := <-stream.ch: - if !ok { - r.Close(id) - return nil, true, nil - } - if item.err != nil { - r.Close(id) - return nil, false, item.err - } - if chunkSize > 0 && len(item.data) > chunkSize { - head := append([]byte(nil), item.data[:chunkSize]...) - stream.pending = append(stream.pending[:0], item.data[chunkSize:]...) - return head, false, nil - } - return item.data, false, nil - case <-stream.ctx.Done(): - return nil, false, stream.ctx.Err() - } -} - -func (s *fetchStream) readLoop() { - defer close(s.ch) - buf := make([]byte, 64*1024) - for { - n, err := s.body.Read(buf) - if n > 0 { - chunk := append([]byte(nil), buf[:n]...) - select { - case s.ch <- fetchChunk{data: chunk}: - case <-s.ctx.Done(): - return - } - } - if err == io.EOF { - return - } - if err != nil { - select { - case s.ch <- fetchChunk{err: err}: - case <-s.ctx.Done(): - } - return - } - } -} - -func (r *fetchRegistry) Close(id string) { - r.mu.Lock() - stream := r.streams[id] - delete(r.streams, id) - r.mu.Unlock() - closeFetchStream(stream) -} - -// CloseAll permanently closes the registry and cancels both active and -// in-flight fetches. It is safe to call more than once. -func (r *fetchRegistry) CloseAll() { - if r == nil { - return - } - r.mu.Lock() - if r.closed { - r.mu.Unlock() - return - } - r.closed = true - if r.cancel != nil { - r.cancel() - } - streams := r.streams - r.streams = make(map[string]*fetchStream) - r.mu.Unlock() - - for _, stream := range streams { - closeFetchStream(stream) - } -} - -func closeFetchStream(stream *fetchStream) { - if stream == nil { - return - } - stream.closeOnce.Do(func() { - if stream.cancel != nil { - stream.cancel() - } - _ = stream.body.Close() - }) -} - -func (r *fetchRegistry) Abort(id string, _ string) { - r.Close(id) -} - -func (r *fetchRegistry) get(id string) *fetchStream { - r.mu.Lock() - defer r.mu.Unlock() - return r.streams[id] -} - -func buildFetchBody(body any) (string, any, error) { - switch v := body.(type) { - case nil: - return "", nil, nil - case string: - return "text/plain;charset=UTF-8", v, nil - case []byte: - return "application/octet-stream", v, nil - case goja.ArrayBuffer: - return "application/octet-stream", v.Bytes(), nil - case *file.File: - return "application/octet-stream", v.Reader, nil - case *fetchFormDataSnapshot: - pr, pw := io.Pipe() - mw := xhr.NewMultipart(pw) - for _, entry := range v.entries { - switch vv := entry.value.(type) { - case string: - if err := mw.WriteField(entry.name, vv); err != nil { - _ = pw.CloseWithError(err) - return "", nil, err - } - case *file.File: - if err := mw.WriteFile(entry.name, vv); err != nil { - _ = pw.CloseWithError(err) - return "", nil, err - } - } - } - go func() { - if err := mw.Send(); err != nil { - _ = pw.CloseWithError(err) - return - } - if err := mw.Close(); err != nil { - _ = pw.CloseWithError(err) - return - } - _ = pw.Close() - }() - return mw.FormDataContentType(), pr, nil - default: - if typed, ok := v.(interface{ Bytes() []byte }); ok { - return "application/octet-stream", typed.Bytes(), nil - } - return "", fmt.Sprint(v), nil - } -} - -func hasHeader(headers [][2]string, key string) bool { - for _, header := range headers { - if strings.EqualFold(header[0], key) { - return true - } - } - return false -} - -func setFetchFingerprint(client *req.Client, fingerprint string) { - switch fingerprint { - case "chrome": - client.ImpersonateChrome() - case "firefox": - client.ImpersonateFirefox() - case "safari": - client.ImpersonateSafari() - } -} - -func errorsAsTimeout(err error, target *net.Error) bool { - return errors.As(err, target) && (*target).Timeout() -} diff --git a/pkg/download/engine/inject/stream/module_integration_test.go b/pkg/download/engine/inject/stream/module_integration_test.go index 6a715149..b2c11cec 100644 --- a/pkg/download/engine/inject/stream/module_integration_test.go +++ b/pkg/download/engine/inject/stream/module_integration_test.go @@ -258,7 +258,6 @@ func waitForJSValue(t *testing.T, engine *engine.Engine, expression, want string func newStreamTestEngine(t *testing.T, created chan<- createdObjectURL) (*engine.Engine, func()) { t.Helper() var next atomic.Int64 - var cleanup func() e := engine.NewEngine(&engine.Config{ StreamConfig: &stream.Config{ CreateObjectURL: func(opts *stream.ObjectURLOptions, open stream.ObjectURLOpener) (string, error) { @@ -266,13 +265,7 @@ func newStreamTestEngine(t *testing.T, created chan<- createdObjectURL) (*engine return fmt.Sprintf("blob:test-%d", next.Add(1)), nil }, RevokeObjectURL: func(string) error { return nil }, - RegisterCleanup: func(fn func()) { - cleanup = fn - }, }, }) - if cleanup == nil { - t.Fatal("stream cleanup was not registered") - } - return e, cleanup + return e, e.Close } diff --git a/pkg/download/engine/inject/stream/module_test.go b/pkg/download/engine/inject/stream/module_test.go deleted file mode 100644 index 793ccbd3..00000000 --- a/pkg/download/engine/inject/stream/module_test.go +++ /dev/null @@ -1,191 +0,0 @@ -package stream - -import ( - "context" - "io" - "strings" - "sync/atomic" - "testing" - - "github.com/GopeedLab/gopeed/pkg/download/engine/inject/file" - "github.com/GopeedLab/gopeed/pkg/download/engine/inject/formdata" - "github.com/dop251/goja" -) - -func TestExportFetchRequestSnapshotsArrayBuffer(t *testing.T) { - runtime := goja.New() - buffer := runtime.NewArrayBuffer([]byte("before")) - request := runtime.NewObject() - setFetchRequestDefaults(t, request) - if err := request.Set("body", buffer); err != nil { - t.Fatal(err) - } - - meta, err := exportFetchRequest(runtime, request) - if err != nil { - t.Fatal(err) - } - buffer.Bytes()[0] = 'X' - - body, ok := meta.Body.([]byte) - if !ok { - t.Fatalf("expected byte snapshot, got %T", meta.Body) - } - if got := string(body); got != "before" { - t.Fatalf("ArrayBuffer snapshot changed after export: %q", got) - } -} - -func TestExportFetchRequestSnapshotsFormData(t *testing.T) { - runtime := goja.New() - runtime.SetFieldNameMapper(goja.TagFieldNameMapper("json", true)) - if err := formdata.Enable(runtime); err != nil { - t.Fatal(err) - } - value, err := runtime.RunString(` - globalThis.form = new FormData(); - form.append("field", "before"); - form; - `) - if err != nil { - t.Fatal(err) - } - request := runtime.NewObject() - setFetchRequestDefaults(t, request) - if err := request.Set("body", value); err != nil { - t.Fatal(err) - } - - meta, err := exportFetchRequest(runtime, request) - if err != nil { - t.Fatal(err) - } - if _, err := runtime.RunString(`form.set("field", "after")`); err != nil { - t.Fatal(err) - } - - body, ok := meta.Body.(*fetchFormDataSnapshot) - if !ok { - t.Fatalf("expected FormData snapshot, got %T", meta.Body) - } - if len(body.entries) != 1 { - t.Fatalf("expected one FormData entry, got %d", len(body.entries)) - } - if got := body.entries[0]; got.name != "field" || got.value != "before" { - t.Fatalf("unexpected FormData snapshot: %#v", got) - } -} - -func setFetchRequestDefaults(t *testing.T, request *goja.Object) { - t.Helper() - for key, value := range map[string]any{ - "url": "https://example.test", - "method": "POST", - "redirect": "follow", - "credentials": "same-origin", - } { - if err := request.Set(key, value); err != nil { - t.Fatal(err) - } - } -} - -func TestSnapshotFetchBodyCopiesFileMetadata(t *testing.T) { - source := &file.File{ - Reader: strings.NewReader("payload"), - Name: "before.txt", - Size: 7, - } - - value, err := snapshotFetchBody(source) - if err != nil { - t.Fatal(err) - } - snapshot, ok := value.(*file.File) - if !ok { - t.Fatalf("expected file snapshot, got %T", value) - } - source.Name = "after.txt" - source.Size = 99 - - if snapshot.Name != "before.txt" || snapshot.Size != 7 { - t.Fatalf("file metadata was not snapshotted: %#v", snapshot) - } -} - -func TestFetchRegistryCloseAll(t *testing.T) { - registry := newFetchRegistry() - streamCtx, cancel := context.WithCancel(context.Background()) - body := &trackingReadCloser{} - registry.streams["active"] = &fetchStream{ - body: body, - cancel: cancel, - ctx: streamCtx, - ch: make(chan fetchChunk), - } - - registry.CloseAll() - registry.CloseAll() - - select { - case <-streamCtx.Done(): - default: - t.Fatal("active fetch context was not canceled") - } - if got := body.closeCalls.Load(); got != 1 { - t.Fatalf("expected body to close once, got %d", got) - } - registry.mu.Lock() - closed := registry.closed - remaining := len(registry.streams) - registry.mu.Unlock() - if !closed || remaining != 0 { - t.Fatalf("registry not fully closed: closed=%v remaining=%d", closed, remaining) - } - if _, err := registry.Open("", nil, &fetchRequest{URL: "https://example.test"}); err == nil { - t.Fatal("expected a closed registry to reject new fetches") - } -} - -func TestFetchRegistryReadPreservesChunkOrder(t *testing.T) { - registry := newFetchRegistry() - ctx, cancel := context.WithCancel(context.Background()) - stream := &fetchStream{ - body: io.NopCloser(strings.NewReader("")), - cancel: cancel, - ctx: ctx, - ch: make(chan fetchChunk, 2), - } - stream.ch <- fetchChunk{data: []byte("abc")} - stream.ch <- fetchChunk{data: []byte("def")} - registry.streams["ordered"] = stream - defer registry.CloseAll() - - var got strings.Builder - for range 4 { - chunk, done, err := registry.Read("ordered", 2) - if err != nil { - t.Fatal(err) - } - if done { - t.Fatal("stream ended before buffered chunks were read") - } - got.Write(chunk) - } - if got.String() != "abcdef" { - t.Fatalf("fetch chunks were reordered: %q", got.String()) - } -} - -type trackingReadCloser struct { - closeCalls atomic.Int32 -} - -func (r *trackingReadCloser) Read([]byte) (int, error) { - return 0, io.EOF -} - -func (r *trackingReadCloser) Close() error { - r.closeCalls.Add(1) - return nil -} diff --git a/pkg/download/engine/inject/stream/stream.js b/pkg/download/engine/inject/stream/stream.js index c8ac6bea..ea3bd85b 100644 --- a/pkg/download/engine/inject/stream/stream.js +++ b/pkg/download/engine/inject/stream/stream.js @@ -1,10 +1,6 @@ (function () { const createBlobObjectURL = globalThis.__gopeed_create_blob_object_url; const revokeObjectURL = globalThis.__gopeed_revoke_object_url; - const fetchOpen = globalThis.__gopeed_fetch_open; - const fetchRead = globalThis.__gopeed_fetch_read; - const fetchClose = globalThis.__gopeed_fetch_close; - const fetchAbort = globalThis.__gopeed_fetch_abort; if (typeof globalThis.ReadableStream === "undefined") { class ReadableStreamDefaultController { @@ -30,7 +26,10 @@ this._stream = stream; } - read() { + read(view) { + if (view !== undefined) { + return this._stream._readInto(view); + } return this._stream._read(); } @@ -59,6 +58,7 @@ this._errored = null; this._reader = null; this._pulling = false; + this._disturbed = false; this.locked = false; this._controller = new ReadableStreamDefaultController(this); if (typeof source.start === "function") { @@ -96,6 +96,7 @@ if (this._errored) { return Promise.reject(this._errored); } + this._disturbed = true; this._markBodyUsed(); if (this._queue.length > 0) { return Promise.resolve({ done: false, value: this._queue.shift() }); @@ -116,6 +117,26 @@ }); } + async _readInto(view) { + if (!ArrayBuffer.isView(view) || view.byteLength === 0) { + throw new TypeError("BYOB reads require a non-empty ArrayBufferView"); + } + const result = await this._read(); + if (result.done) { + return { done: true, value: new Uint8Array(view.buffer, view.byteOffset, 0) }; + } + if (!(result.value instanceof Uint8Array)) { + throw new TypeError("Byte streams require Uint8Array chunks"); + } + const length = Math.min(view.byteLength, result.value.byteLength); + const target = new Uint8Array(view.buffer, view.byteOffset, length); + target.set(result.value.subarray(0, length)); + if (length < result.value.byteLength) { + this._queue.unshift(result.value.subarray(length)); + } + return { done: false, value: target }; + } + _maybePull() { if (this._closed || this._errored || this._pulling) { return; @@ -155,6 +176,7 @@ } cancel(reason) { + this._disturbed = true; this._markBodyUsed(); if (this._source && typeof this._source.cancel === "function") { return Promise.resolve(this._source.cancel(reason)).then(() => { @@ -185,6 +207,69 @@ writer.releaseLock(); } } + + tee() { + const reader = this.getReader(); + let reading = false; + let currentRead = null; + let finished = false; + const waiting = [[], []]; + const queues = [[], []]; + + const cloneChunk = (chunk) => { + if (chunk instanceof ArrayBuffer) return chunk.slice(0); + if (ArrayBuffer.isView(chunk)) { + const buffer = chunk.buffer.slice(0); + if (chunk instanceof DataView) return new DataView(buffer, chunk.byteOffset, chunk.byteLength); + return new chunk.constructor(buffer, chunk.byteOffset, chunk.length); + } + return chunk; + }; + const flush = (index) => { + while (waiting[index].length && queues[index].length) { + waiting[index].shift().enqueue(queues[index].shift()); + } + if (finished && queues[index].length === 0) { + while (waiting[index].length) waiting[index].shift().close(); + } + }; + const pump = () => { + if (reading) return currentRead; + if (finished) return Promise.resolve(); + reading = true; + currentRead = (async () => { + try { + const result = await reader.read(); + if (result.done) { + finished = true; + } else { + queues[0].push(result.value); + queues[1].push(cloneChunk(result.value)); + } + } catch (error) { + finished = true; + for (const branches of waiting) { + while (branches.length) branches.shift().error(error); + } + throw error; + } finally { + reading = false; + currentRead = null; + flush(0); + flush(1); + } + })(); + return currentRead; + }; + const branch = (index) => new ReadableStream({ + pull(controller) { + waiting[index].push(controller); + flush(index); + return pump(); + }, + }); + return [branch(0), branch(1)]; + } } globalThis.ReadableStream = ReadableStream; @@ -351,167 +436,6 @@ }; } - function toUint8Array(chunk) { - if (chunk == null) { - return new Uint8Array(0); - } - if (chunk instanceof Uint8Array) { - return chunk; - } - if (typeof chunk === "string") { - return new TextEncoder().encode(chunk); - } - if (chunk instanceof ArrayBuffer) { - return new Uint8Array(chunk); - } - if (typeof ArrayBuffer !== "undefined" && ArrayBuffer.isView && ArrayBuffer.isView(chunk)) { - return new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength); - } - if (typeof Blob !== "undefined" && chunk instanceof Blob) { - if (chunk._buffer instanceof Uint8Array) { - return chunk._buffer; - } - } - return new Uint8Array(0); - } - - function createBodyReadableStream(owner) { - return new ReadableStream({ - start(controller) { - Promise.resolve().then(async () => { - owner.bodyUsed = true; - if (owner._noBody) { - controller.close(); - return; - } - if (owner._bodyArrayBuffer) { - controller.enqueue(toUint8Array(owner._bodyArrayBuffer)); - controller.close(); - return; - } - if (owner._bodyBlob) { - const data = await owner._bodyBlob.arrayBuffer(); - controller.enqueue(new Uint8Array(data)); - controller.close(); - return; - } - if (owner._bodyText != null) { - controller.enqueue(toUint8Array(owner._bodyText)); - controller.close(); - return; - } - if (owner._bodyInit != null) { - controller.enqueue(toUint8Array(owner._bodyInit)); - } - controller.close(); - }).catch((err) => { - controller.error(err); - }); - } - }); - } - - async function readAllFromStream(stream, asText) { - if (!stream) { - return asText ? "" : new Uint8Array(0); - } - const reader = stream.getReader(); - const chunks = []; - let total = 0; - try { - while (true) { - const { done, value } = await reader.read(); - if (done) { - break; - } - const chunk = toUint8Array(value); - chunks.push(chunk); - total += chunk.byteLength; - } - } finally { - reader.releaseLock(); - } - const merged = new Uint8Array(total); - let offset = 0; - for (const chunk of chunks) { - merged.set(chunk, offset); - offset += chunk.byteLength; - } - if (asText) { - return new TextDecoder().decode(merged); - } - return merged; - } - - function attachResponseStreaming(response, stream) { - response.__gopeedBodyStream = stream; - response.__gopeedBodyConsumed = false; - const ensureUnused = () => { - if (response.__gopeedBodyConsumed) { - throw new TypeError("Already read"); - } - }; - const markBodyUsed = () => { - ensureUnused(); - response.__gopeedBodyConsumed = true; - response.bodyUsed = true; - }; - if (stream) { - stream.__gopeedMarkBodyUsed = () => { - if (!response.__gopeedBodyConsumed) { - response.__gopeedBodyConsumed = true; - response.bodyUsed = true; - } - }; - } - response.text = async function () { - ensureUnused(); - markBodyUsed(); - return readAllFromStream(stream, true); - }; - response.arrayBuffer = async function () { - ensureUnused(); - markBodyUsed(); - const bytes = await readAllFromStream(stream, false); - return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength); - }; - response.blob = async function () { - ensureUnused(); - markBodyUsed(); - const contentType = this.headers && this.headers.get ? (this.headers.get("content-type") || "") : ""; - // Response.blob() is a materializing body consumer. Keeping the live - // response stream behind an empty one-shot Blob makes retry, range and a - // second object-URL reader silently produce an empty file. - const bytes = await readAllFromStream(stream, false); - return new Blob([bytes], { type: contentType }); - }; - response.json = async function () { - const text = await this.text(); - return JSON.parse(text); - }; - return response; - } - - if (typeof globalThis.Response === "function") { - const responseProto = globalThis.Response.prototype; - const bodyDescriptor = Object.getOwnPropertyDescriptor(responseProto, "body"); - if (!bodyDescriptor || typeof bodyDescriptor.get !== "function") { - Object.defineProperty(responseProto, "body", { - configurable: true, - enumerable: true, - get() { - if (this.__gopeedBodyStream) { - return this.__gopeedBodyStream; - } - if (!this.__gopeedBodyStream) { - this.__gopeedBodyStream = createBodyReadableStream(this); - } - return this.__gopeedBodyStream; - } - }); - } - } - const blobReaders = new Map(); function getValueTypeName(value) { @@ -883,82 +807,4 @@ } }; - const originalFetch = typeof globalThis.fetch === "function" - ? globalThis.fetch.bind(globalThis) - : null; - - if (typeof fetchOpen === "function") { - globalThis.fetch = async function (input, init) { - const request = new Request(input, init); - if (originalFetch && (request.method === "HEAD" || request.redirect === "manual" || request.redirect === "error")) { - return originalFetch(input, init); - } - let body = null; - if (request._bodyFormData) { - body = request._bodyFormData; - } else if (request._bodyArrayBuffer) { - body = request._bodyArrayBuffer; - } else if (request._bodyBlob) { - body = await request._bodyBlob.arrayBuffer(); - } else if (request._bodyInit != null && typeof request._bodyInit === "object") { - body = request._bodyInit; - } else if (request._bodyText != null) { - body = request._bodyText; - } else if (request._bodyInit != null) { - body = request._bodyInit; - } - const headers = []; - request.headers.forEach((value, key) => { - headers.push([key, value]); - }); - let meta; - try { - meta = await fetchOpen({ - url: request.url, - method: request.method, - headers, - body, - redirect: request.redirect, - credentials: request.credentials - }); - } catch (error) { - throw error instanceof Error ? error : new TypeError(String(error)); - } - const stream = new ReadableStream({ - async pull(controller) { - let chunk; - try { - chunk = await fetchRead(meta.id, 64 * 1024); - } catch (error) { - fetchClose(meta.id); - controller.error(error instanceof Error ? error : new TypeError(String(error))); - return; - } - if (chunk == null) { - fetchClose(meta.id); - controller.close(); - return; - } - const bytes = chunk instanceof Uint8Array ? chunk : new Uint8Array(chunk); - if (bytes.byteLength === 0) { - fetchClose(meta.id); - controller.close(); - return; - } - controller.enqueue(bytes); - }, - cancel(reason) { - fetchAbort(meta.id, reason == null ? "" : String(reason)); - } - }); - const response = new Response(null, { - status: meta.status, - statusText: meta.statusText, - headers: meta.headers, - url: meta.url - }); - return attachResponseStreaming(response, stream); - }; - globalThis.fetch.__gopeedOriginalFetch = originalFetch; - } })(); diff --git a/pkg/download/engine/inject/xhr/module.go b/pkg/download/engine/inject/xhr/module.go index 98b59567..3224e68f 100644 --- a/pkg/download/engine/inject/xhr/module.go +++ b/pkg/download/engine/inject/xhr/module.go @@ -1,482 +1,15 @@ package xhr import ( - "bytes" - "errors" - "io" - "mime/multipart" - "net" - "net/http" - "net/url" - "strings" - "time" + _ "embed" - "github.com/GopeedLab/gopeed/pkg/download/engine/inject/file" - "github.com/GopeedLab/gopeed/pkg/download/engine/inject/formdata" - "github.com/GopeedLab/gopeed/pkg/download/engine/util" "github.com/dop251/goja" - "github.com/imroc/req/v3" ) -const ( - eventLoad = "load" - eventReadystatechange = "readystatechange" - eventProgress = "progress" - eventAbort = "abort" - eventError = "error" - eventTimeout = "timeout" -) +//go:embed xhr.js +var script string -const ( - redirectError = "error" - redirectFollow = "follow" - redirectManual = "manual" -) - -type ProgressEvent struct { - Type string `json:"type"` - LengthComputable bool `json:"lengthComputable"` - Loaded int64 `json:"loaded"` - Total int64 `json:"total"` -} - -type EventProp struct { - eventListeners map[string]func(event *ProgressEvent) - Onload func(event *ProgressEvent) `json:"onload"` - Onprogress func(event *ProgressEvent) `json:"onprogress"` - Onabort func(event *ProgressEvent) `json:"onabort"` - Onerror func(event *ProgressEvent) `json:"onerror"` - Ontimeout func(event *ProgressEvent) `json:"ontimeout"` -} - -func (ep *EventProp) AddEventListener(event string, cb func(event *ProgressEvent)) { - ep.eventListeners[event] = cb -} - -func (ep *EventProp) RemoveEventListener(event string) { - delete(ep.eventListeners, event) -} - -func (ep *EventProp) callOnload() { - event := &ProgressEvent{ - Type: eventLoad, - LengthComputable: false, - } - if ep.Onload != nil { - ep.Onload(event) - } - ep.callEventListener(event) -} - -func (ep *EventProp) callOnprogress(loaded, total int64) { - event := &ProgressEvent{ - Type: eventProgress, - LengthComputable: true, - Loaded: loaded, - Total: total, - } - if ep.Onprogress != nil { - ep.Onprogress(event) - } - ep.callEventListener(event) -} - -func (ep *EventProp) callOnabort() { - event := &ProgressEvent{ - Type: eventAbort, - LengthComputable: false, - } - if ep.Onabort != nil { - ep.Onabort(event) - } - ep.callEventListener(event) -} - -func (ep *EventProp) callOnerror() { - event := &ProgressEvent{ - Type: eventError, - LengthComputable: false, - } - if ep.Onerror != nil { - ep.Onerror(event) - } - ep.callEventListener(event) -} - -func (ep *EventProp) callOntimeout() { - event := &ProgressEvent{ - Type: eventTimeout, - LengthComputable: false, - } - if ep.Ontimeout != nil { - ep.Ontimeout(event) - } - ep.callEventListener(event) -} - -func (ep *EventProp) callEventListener(event *ProgressEvent) { - if cb, ok := ep.eventListeners[event.Type]; ok { - cb(event) - } -} - -type XMLHttpRequestUpload struct { - *EventProp -} - -type XMLHttpRequest struct { - method string - url string - requestHeaders http.Header - responseHeaders http.Header - aborted bool - client *req.Client - fingerprint string - - WithCredentials bool `json:"withCredentials"` - Upload *XMLHttpRequestUpload `json:"upload"` - Timeout int `json:"timeout"` - ReadyState int `json:"readyState"` - Status int `json:"status"` - StatusText string `json:"statusText"` - Response string `json:"response"` - ResponseText string `json:"responseText"` - // https://developer.mozilla.org/zh-CN/docs/Web/API/XMLHttpRequest/responseURL - // https://xhr.spec.whatwg.org/#the-responseurl-attribute - ResponseUrl string `json:"responseURL"` - // extend fetch redirect - // https://developer.mozilla.org/en-US/docs/Web/API/RequestInit#redirect - // https://fetch.spec.whatwg.org/#concept-request-redirect-mode - Redirect string `json:"redirect"` - *EventProp - Onreadystatechange func(event *ProgressEvent) `json:"onreadystatechange"` -} - -func (xhr *XMLHttpRequest) Open(method, url string) { - xhr.method = method - xhr.url = url - xhr.requestHeaders = make(http.Header) - xhr.responseHeaders = make(http.Header) - xhr.doReadystatechange(1) -} - -func (xhr *XMLHttpRequest) SetRequestHeader(key, value string) { - xhr.requestHeaders.Add(key, value) -} - -func (xhr *XMLHttpRequest) Send(data goja.Value) { - setFingerprint(xhr.client, xhr.fingerprint) - - d := xhr.parseData(data) - var ( - contentType string - contentLength int64 - isStringBody bool - ) - - // Create request using req library - reqBuilder := xhr.client.R() - - // Set headers first - if xhr.requestHeaders != nil { - for key, values := range xhr.requestHeaders { - if len(values) > 0 { - // Merge multiple values with comma separator (HTTP standard) - mergedValue := strings.Join(values, ", ") - reqBuilder.SetHeader(key, mergedValue) - } - } - } - - // Handle request body - if d != nil && xhr.method != "GET" && xhr.method != "HEAD" { - switch v := d.(type) { - case string: - reqBuilder.SetBody(v) - contentType = "text/plain;charset=UTF-8" - contentLength = int64(len(v)) - isStringBody = true - case *file.File: - reqBuilder.SetBody(v.Reader) - contentType = "application/octet-stream" - contentLength = v.Size - case *formdata.FormData: - pr, pw := io.Pipe() - mw := NewMultipart(pw) - for _, e := range v.Entries() { - arr := e.([]any) - k := arr[0].(string) - v := arr[1] - switch v := v.(type) { - case string: - mw.WriteField(k, v) - case *file.File: - mw.WriteFile(k, v) - } - } - go func() { - defer pw.Close() - defer mw.Close() - mw.Send() - }() - reqBuilder.SetBody(pr) - contentType = mw.FormDataContentType() - contentLength = mw.Size() - } - } - - // Only string body can specify Content-Type header by user - if contentType != "" && (!isStringBody || xhr.requestHeaders.Get("Content-Type") == "") { - reqBuilder.SetHeader("Content-Type", contentType) - } - - // Set timeout - if xhr.Timeout > 0 { - xhr.client.SetTimeout(time.Duration(xhr.Timeout) * time.Millisecond) - } - - // Configure redirect behavior - xhr.client.SetRedirectPolicy(func(req *http.Request, via []*http.Request) error { - if xhr.Redirect == redirectManual { - return http.ErrUseLastResponse - } - if xhr.Redirect == redirectError { - return errors.New("redirect failed") - } - if len(via) > 20 { - return errors.New("too many redirects") - } - return nil - }) - - // Execute request - resp, err := reqBuilder.Send(xhr.method, xhr.url) - if err != nil { - // handle timeout error - var ne net.Error - if errors.As(err, &ne) && ne.Timeout() { - if xhr.Timeout > 0 { - xhr.Upload.callOntimeout() - xhr.callOntimeout() - } - return - } - xhr.Upload.callOnerror() - xhr.callOnerror() - return - } - - xhr.Upload.callOnprogress(contentLength, contentLength) - if !xhr.aborted { - xhr.Upload.callOnload() - } - - // Set response URL (final URL after redirects) - if resp.Response != nil && resp.Response.Request != nil && resp.Response.Request.URL != nil { - responseUrl := resp.Response.Request.URL - responseUrl.Fragment = "" - xhr.ResponseUrl = responseUrl.String() - } else { - xhr.ResponseUrl = xhr.url - } - - // Set response headers - xhr.responseHeaders = resp.Header - xhr.Status = resp.StatusCode - xhr.StatusText = resp.Status - xhr.doReadystatechange(2) - - bodyBytes := resp.Bytes() - xhr.doReadystatechange(3) - xhr.Response = string(bodyBytes) - xhr.ResponseText = xhr.Response - xhr.doReadystatechange(4) - respBodyLen := int64(len(bodyBytes)) - xhr.callOnprogress(respBodyLen, respBodyLen) - if !xhr.aborted { - xhr.callOnload() - } -} - -func (xhr *XMLHttpRequest) Abort() { - xhr.doReadystatechange(0) - xhr.aborted = true - xhr.Upload.callOnabort() - xhr.callOnabort() -} - -func (xhr *XMLHttpRequest) GetResponseHeader(key string) string { - if xhr.responseHeaders == nil { - return "" - } - return strings.Join(xhr.responseHeaders.Values(key), ", ") -} - -func (xhr *XMLHttpRequest) GetAllResponseHeaders() string { - var buf bytes.Buffer - for k, v := range xhr.responseHeaders { - buf.WriteString(k) - buf.WriteString(": ") - buf.WriteString(strings.Join(v, ", ")) - buf.WriteString("\r\n") - } - return buf.String() -} - -func (xhr *XMLHttpRequest) callOnreadystatechange() { - event := &ProgressEvent{ - Type: eventReadystatechange, - LengthComputable: false, - } - if xhr.Onreadystatechange != nil { - xhr.Onreadystatechange(event) - } - xhr.callEventListener(event) -} - -func (xhr *XMLHttpRequest) doReadystatechange(state int) { - if xhr.aborted { - return - } - xhr.ReadyState = state - xhr.callOnreadystatechange() -} - -// parse js data to go struct -func (xhr *XMLHttpRequest) parseData(data goja.Value) any { - // check if data is null or undefined - if data == nil || goja.IsNull(data) || goja.IsUndefined(data) || goja.IsNaN(data) { - return nil - } - // check if data is File - f, ok := data.Export().(*file.File) - if ok { - return f - } - // check if data is FormData - fd, ok := data.Export().(*formdata.FormData) - if ok { - return fd - } - // otherwise, return data as string - return data.String() -} - -func Enable(runtime *goja.Runtime, proxyHandler func(r *http.Request) (*url.URL, error)) error { - progressEvent := runtime.ToValue(func(call goja.ConstructorCall) *goja.Object { - if len(call.Arguments) < 1 { - util.ThrowTypeError(runtime, "Failed to construct 'ProgressEvent': 1 argument required, but only 0 present.") - } - instance := &ProgressEvent{ - Type: call.Argument(0).String(), - } - instanceValue := runtime.ToValue(instance).(*goja.Object) - instanceValue.SetPrototype(call.This.Prototype()) - return instanceValue - }) - xhr := runtime.ToValue(func(call goja.ConstructorCall) *goja.Object { - // Create req client with proxy support - client := req.C() - if proxyHandler != nil { - client.SetProxy(proxyHandler) - } - - instance := &XMLHttpRequest{ - client: client, - fingerprint: util.SafeGet[string](runtime, FingerprintMagicKey), - Upload: &XMLHttpRequestUpload{ - EventProp: &EventProp{ - eventListeners: make(map[string]func(event *ProgressEvent)), - }, - }, - EventProp: &EventProp{ - eventListeners: make(map[string]func(event *ProgressEvent)), - }, - } - instanceValue := runtime.ToValue(instance).(*goja.Object) - instanceValue.SetPrototype(call.This.Prototype()) - return instanceValue - }) - runtime.Set("__gopeed_setFingerprint", func(fingerprint string) { - runtime.Set(FingerprintMagicKey, fingerprint) - }) - if err := runtime.Set("ProgressEvent", progressEvent); err != nil { - return err - } - if err := runtime.Set("XMLHttpRequest", xhr); err != nil { - return err - } - return nil -} - -// Wrap multipart.Writer and stat content length -type multipartWrapper struct { - statBuffer *bytes.Buffer - statWriter *multipart.Writer - writer *multipart.Writer - fields map[string]any -} - -func NewMultipart(w io.Writer) *multipartWrapper { - var buf bytes.Buffer - return &multipartWrapper{ - statBuffer: &buf, - statWriter: multipart.NewWriter(&buf), - writer: multipart.NewWriter(w), - fields: make(map[string]any), - } -} - -func (w *multipartWrapper) WriteField(fieldname string, value string) error { - w.fields[fieldname] = value - return w.statWriter.WriteField(fieldname, value) -} - -func (w *multipartWrapper) WriteFile(fieldname string, file *file.File) error { - w.fields[fieldname] = file - _, err := w.statWriter.CreateFormFile(fieldname, file.Name) - if err != nil { - return err - } - return nil -} - -func (w *multipartWrapper) Size() int64 { - w.statWriter.Close() - size := int64(w.statBuffer.Len()) - for _, v := range w.fields { - switch v := v.(type) { - case *file.File: - size += v.Size - } - } - return size -} - -func (w *multipartWrapper) Send() error { - for k, v := range w.fields { - switch v := v.(type) { - case string: - if err := w.writer.WriteField(k, v); err != nil { - return err - } - case *file.File: - fw, err := w.writer.CreateFormFile(k, v.Name) - if err != nil { - return err - } - if _, err = io.Copy(fw, v); err != nil { - return err - } - } - } - return nil -} - -func (w *multipartWrapper) FormDataContentType() string { - return w.writer.FormDataContentType() -} - -func (w *multipartWrapper) Close() error { - return w.writer.Close() +func Enable(runtime *goja.Runtime) error { + _, err := runtime.RunString(script) + return err } diff --git a/pkg/download/engine/inject/xhr/tls_fingerprint.go b/pkg/download/engine/inject/xhr/tls_fingerprint.go deleted file mode 100644 index 664ba462..00000000 --- a/pkg/download/engine/inject/xhr/tls_fingerprint.go +++ /dev/null @@ -1,24 +0,0 @@ -package xhr - -import "github.com/imroc/req/v3" - -type Fingerprint string - -const ( - FingerprintMagicKey = "__gopeed_xhr_fingerprint" - - fingerprintChrome = "chrome" - fingerprintFirefox = "firefox" - fingerprintSafari = "safari" -) - -func setFingerprint(client *req.Client, fingerprint string) { - switch fingerprint { - case fingerprintChrome: - client.ImpersonateChrome() - case fingerprintFirefox: - client.ImpersonateFirefox() - case fingerprintSafari: - client.ImpersonateSafari() - } -} diff --git a/pkg/download/engine/inject/xhr/xhr.js b/pkg/download/engine/inject/xhr/xhr.js new file mode 100644 index 00000000..331b725a --- /dev/null +++ b/pkg/download/engine/inject/xhr/xhr.js @@ -0,0 +1,302 @@ +(function () { + if (typeof globalThis.ProgressEvent === "undefined") { + globalThis.ProgressEvent = class ProgressEvent extends Event { + constructor(type, init) { + super(type); + init = init || {}; + this.lengthComputable = Boolean(init.lengthComputable); + this.loaded = Number(init.loaded) || 0; + this.total = Number(init.total) || 0; + } + }; + } + + function createProgressEvent(type, loaded, total) { + const lengthComputable = Number.isFinite(total) && total >= 0; + return new ProgressEvent(type, { + lengthComputable, + loaded: loaded || 0, + total: lengthComputable ? total : 0, + }); + } + + function canonicalHeaderName(name) { + return String(name).toLowerCase().split("-").map(function (part) { + return part ? part.charAt(0).toUpperCase() + part.slice(1) : part; + }).join("-"); + } + + function bodySize(body) { + if (body == null) return 0; + if (typeof body === "string") return new TextEncoder().encode(body).byteLength; + if (body instanceof ArrayBuffer) return body.byteLength; + if (typeof ArrayBuffer !== "undefined" && ArrayBuffer.isView && ArrayBuffer.isView(body)) { + return body.byteLength; + } + if (typeof Blob !== "undefined" && body instanceof Blob) return Number(body.size) || 0; + if (body && Number.isFinite(Number(body.size))) return Number(body.size); + return 0; + } + + class XMLHttpRequestUpload extends EventTarget { + constructor() { + super(); + this.onloadstart = null; + this.onprogress = null; + this.onabort = null; + this.onerror = null; + this.onload = null; + this.ontimeout = null; + this.onloadend = null; + } + + _emit(type, loaded, total) { + const event = createProgressEvent(type, loaded, total); + const handler = this["on" + type]; + if (typeof handler === "function") handler.call(this, event); + this.dispatchEvent(event); + } + } + + class XMLHttpRequest extends EventTarget { + constructor() { + super(); + this.readyState = XMLHttpRequest.UNSENT; + this.response = ""; + this.responseText = ""; + this.responseType = ""; + this.responseURL = ""; + this.responseXML = null; + this.status = 0; + this.statusText = ""; + this.timeout = 0; + this.withCredentials = false; + this.upload = new XMLHttpRequestUpload(); + this.redirect = "follow"; + this.onreadystatechange = null; + this.onloadstart = null; + this.onprogress = null; + this.onabort = null; + this.onerror = null; + this.onload = null; + this.ontimeout = null; + this.onloadend = null; + this._method = "GET"; + this._url = ""; + this._requestHeaders = new Headers(); + this._responseHeaders = null; + this._controller = null; + this._sendStarted = false; + this._aborted = false; + this._timedOut = false; + this._timeoutID = null; + } + + open(method, url, async) { + if (arguments.length < 2) { + throw new TypeError("XMLHttpRequest.open requires at least 2 arguments"); + } + if (async === false) { + throw new DOMException("Synchronous XMLHttpRequest is not supported", "NotSupportedError"); + } + this._method = String(method).toUpperCase(); + this._url = String(url); + this._requestHeaders = new Headers(); + this._responseHeaders = null; + this._sendStarted = false; + this._aborted = false; + this._timedOut = false; + this.response = ""; + this.responseText = ""; + this.responseURL = ""; + this.status = 0; + this.statusText = ""; + this._setReadyState(XMLHttpRequest.OPENED); + } + + setRequestHeader(name, value) { + if (this.readyState !== XMLHttpRequest.OPENED || this._sendStarted) { + throw new DOMException("XMLHttpRequest is not opened", "InvalidStateError"); + } + this._requestHeaders.append(name, value); + } + + getResponseHeader(name) { + if (!this._responseHeaders || this.readyState < XMLHttpRequest.HEADERS_RECEIVED) return null; + return this._responseHeaders.get(name); + } + + getAllResponseHeaders() { + if (!this._responseHeaders || this.readyState < XMLHttpRequest.HEADERS_RECEIVED) return ""; + let result = ""; + this._responseHeaders.forEach(function (value, name) { + result += canonicalHeaderName(name) + ": " + value + "\r\n"; + }); + return result; + } + + overrideMimeType() { + // Kept as a compatibility no-op until MIME decoding is configurable. + } + + send(body) { + if (this.readyState !== XMLHttpRequest.OPENED || this._sendStarted) { + throw new DOMException("XMLHttpRequest is not opened", "InvalidStateError"); + } + this._sendStarted = true; + this._aborted = false; + this._timedOut = false; + this._controller = new AbortController(); + const uploadTotal = bodySize(body); + this._emit("loadstart", 0, -1); + this.upload._emit("loadstart", 0, uploadTotal); + + if (this.timeout > 0) { + this._timeoutID = setTimeout(() => { + this._timedOut = true; + this._controller.abort(); + }, this.timeout); + } + + const options = { + method: this._method, + headers: this._requestHeaders, + redirect: this.redirect || "follow", + credentials: this.withCredentials ? "include" : "same-origin", + signal: this._controller.signal, + }; + if (this._method !== "GET" && this._method !== "HEAD" && body != null) { + options.body = body; + } + + Promise.resolve().then(() => globalThis.fetch(this._url, options)).then(async (response) => { + if (this._aborted || this._timedOut) return; + this.upload._emit("progress", uploadTotal, uploadTotal); + this.upload._emit("load", uploadTotal, uploadTotal); + this.upload._emit("loadend", uploadTotal, uploadTotal); + + this.status = response.status; + this.statusText = response.statusText; + this.responseURL = response.url || this._url; + this._responseHeaders = response.headers; + this._setReadyState(XMLHttpRequest.HEADERS_RECEIVED); + + const contentLengthValue = response.headers.get("content-length"); + const contentLength = contentLengthValue == null ? -1 : Number(contentLengthValue); + const chunks = []; + let loaded = 0; + if (response.body) { + const reader = response.body.getReader(); + try { + while (true) { + const item = await reader.read(); + if (item.done) break; + const chunk = item.value instanceof Uint8Array ? item.value : new Uint8Array(item.value); + chunks.push(chunk); + loaded += chunk.byteLength; + this._setReadyState(XMLHttpRequest.LOADING); + this._emit("progress", loaded, contentLength); + } + } finally { + if (typeof reader.releaseLock === "function") reader.releaseLock(); + } + } + + const bytes = new Uint8Array(loaded); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + const text = new TextDecoder().decode(bytes); + switch (this.responseType) { + case "arraybuffer": + this.response = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength); + break; + case "blob": + this.response = new Blob([bytes], { type: response.headers.get("content-type") || "" }); + break; + case "json": + try { + this.response = text === "" ? null : JSON.parse(text); + } catch (_) { + this.response = null; + } + break; + case "": + case "text": + default: + this.response = text; + this.responseText = text; + break; + } + this._clearTimeout(); + this._setReadyState(XMLHttpRequest.DONE); + this._emit("load", loaded, contentLength); + this._emit("loadend", loaded, contentLength); + }).catch((error) => { + this._clearTimeout(); + if (this._timedOut) { + this.status = 0; + this._setReadyState(XMLHttpRequest.DONE); + this.upload._emit("timeout", 0, uploadTotal); + this._emit("timeout", 0, -1); + this._emit("loadend", 0, -1); + return; + } + if (this._aborted || (error && error.name === "AbortError")) { + this.status = 0; + this._setReadyState(XMLHttpRequest.DONE); + this.upload._emit("abort", 0, uploadTotal); + this._emit("abort", 0, -1); + this._emit("loadend", 0, -1); + return; + } + this.status = 0; + this._setReadyState(XMLHttpRequest.DONE); + this.upload._emit("error", 0, uploadTotal); + this._emit("error", 0, -1); + this._emit("loadend", 0, -1); + }); + } + + abort() { + if (!this._sendStarted || this.readyState === XMLHttpRequest.DONE) return; + this._aborted = true; + this._clearTimeout(); + if (this._controller) this._controller.abort(); + } + + _clearTimeout() { + if (this._timeoutID != null) { + clearTimeout(this._timeoutID); + this._timeoutID = null; + } + } + + _setReadyState(state) { + this.readyState = state; + this._emit("readystatechange", 0, -1); + } + + _emit(type, loaded, total) { + const event = createProgressEvent(type, loaded, total); + const handler = this["on" + type]; + if (typeof handler === "function") handler.call(this, event); + this.dispatchEvent(event); + } + } + + XMLHttpRequest.UNSENT = 0; + XMLHttpRequest.OPENED = 1; + XMLHttpRequest.HEADERS_RECEIVED = 2; + XMLHttpRequest.LOADING = 3; + XMLHttpRequest.DONE = 4; + XMLHttpRequest.prototype.UNSENT = 0; + XMLHttpRequest.prototype.OPENED = 1; + XMLHttpRequest.prototype.HEADERS_RECEIVED = 2; + XMLHttpRequest.prototype.LOADING = 3; + XMLHttpRequest.prototype.DONE = 4; + + globalThis.XMLHttpRequest = XMLHttpRequest; +})(); diff --git a/pkg/download/engine/polyfill/out/index.js b/pkg/download/engine/polyfill/out/index.js index 916e3edd..77a6cf21 100644 --- a/pkg/download/engine/polyfill/out/index.js +++ b/pkg/download/engine/polyfill/out/index.js @@ -1 +1 @@ -(()=>{var e={847(e,t,r){var n,o;n=function(){"use strict";function e(e,t){(null==t||t>e.length)&&(t=e.length);for(var r=0,n=Array(t);r=t.length?{done:!0}:{done:!1,value:t[o++]}},e:function(e){throw e},f:i}}throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.")}var a,s=!0,l=!1;return{s:function(){n=n.call(t)},n:function(){var e=n.next();return s=e.done,e},e:function(e){l=!0,a=e},f:function(){try{s||null==n.return||n.return()}finally{if(l)throw a}}}}function s(){return s="undefined"!=typeof Reflect&&Reflect.get?Reflect.get.bind():function(e,t,r){var n=function(e,t){for(;!{}.hasOwnProperty.call(e,t)&&null!==(e=l(e)););return e}(e,t);if(n){var o=Object.getOwnPropertyDescriptor(n,t);return o.get?o.get.call(arguments.length<3?e:r):o.value}},s.apply(null,arguments)}function l(e){return l=Object.setPrototypeOf?Object.getPrototypeOf.bind():function(e){return e.__proto__||Object.getPrototypeOf(e)},l(e)}function c(){try{var e=!Boolean.prototype.valueOf.call(Reflect.construct(Boolean,[],function(){}))}catch(e){}return(c=function(){return!!e})()}function u(e,t){return u=Object.setPrototypeOf?Object.setPrototypeOf.bind():function(e,t){return e.__proto__=t,e},u(e,t)}function f(e){var t=function(e,t){if("object"!=typeof e||!e)return e;var r=e[Symbol.toPrimitive];if(void 0!==r){var n=r.call(e,t||"default");if("object"!=typeof n)return n;throw new TypeError("@@toPrimitive must return a primitive value.")}return("string"===t?String:Number)(e)}(e,"string");return"symbol"==typeof t?t:t+""}!function(e){e.AbortSignal,e.AbortController}("undefined"!=typeof self?self:r.g);var p=function(){return i(function e(){n(this,e),Object.defineProperty(this,"listeners",{value:{},writable:!0,configurable:!0})},[{key:"addEventListener",value:function(e,t,r){e in this.listeners||(this.listeners[e]=[]),this.listeners[e].push({callback:t,options:r})}},{key:"removeEventListener",value:function(e,t){if(e in this.listeners)for(var r=this.listeners[e],n=0,o=r.length;n>1)+7),s=new n(a>>3<<3);t=55296&&l<=56319){if(t=55296&&l<=56319)continue}if(o+4>s.length){a+=8,a=(a*=1+t/e.length*2)>>3<<3;var u=new Uint8Array(a);u.set(s),s=u}if(4294967168&l){if(4294965248&l)if(4294901760&l){if(4292870144&l)continue;s[o++]=l>>18&7|240,s[o++]=l>>12&63|128,s[o++]=l>>6&63|128}else s[o++]=l>>12&15|224,s[o++]=l>>6&63|128;else s[o++]=l>>6&31|192;s[o++]=63&l|128}else s[o++]=l}return s.slice(0,o)},y="function"==typeof TextDecoder?TextDecoder.prototype.decode.bind(new TextDecoder):function(e){for(var t=e.length,r=[],n=0;n239?4:l>223?3:l>191?2:1;if(n+u<=t)switch(u){case 1:l<128&&(c=l);break;case 2:128==(192&(o=e[n+1]))&&(s=(31&l)<<6|63&o)>127&&(c=s);break;case 3:o=e[n+1],i=e[n+2],128==(192&o)&&128==(192&i)&&(s=(15&l)<<12|(63&o)<<6|63&i)>2047&&(s<55296||s>57343)&&(c=s);break;case 4:o=e[n+1],i=e[n+2],a=e[n+3],128==(192&o)&&128==(192&i)&&128==(192&a)&&(s=(15&l)<<18|(63&o)<<12|(63&i)<<6|63&a)>65535&&s<1114112&&(c=s)}null===c?(c=65533,u=1):c>65535&&(c-=65536,r.push(c>>>10&1023|55296),c=56320|1023&c),r.push(c),n+=u}for(var f=r.length,p="",d=0;d>2,u=(3&o)<<4|a>>4,f=(15&a)<<2|l>>6,p=63&l;s||(p=64,i||(f=64)),r.push(t[c],t[u],t[f],t[p])}return r.join("")}var s=Object.create||function(e){function t(){}return t.prototype=e,new t};function l(e){return Object.prototype.toString.call(e).slice(8,-1)}function c(e,t){return"object"==typeof e&&Object.prototype.isPrototypeOf.call(e.prototype,t)}function u(e){return"DataView"===l(e)||c(i.DataView,e)}var f=["Int8Array","Uint8Array","Uint8ClampedArray","Int16Array","Uint16Array","Int32Array","Uint32Array","Float32Array","Float64Array","ArrayBuffer"];function p(e){return t=f,r=l(e),-1!==t.indexOf(r)||c(i.ArrayBuffer,e);var t,r}function d(e,r){r=null==r?{}:r;for(var n=0,o=(e=e?e.slice():[]).length;n=t.size&&r.close()})}})}}catch(e){try{new ReadableStream({}),g=function(e){var t=0;return new ReadableStream({pull:function(r){return e.slice(t,t+524288).arrayBuffer().then(function(n){t+=n.byteLength;var o=new Uint8Array(n);r.enqueue(o),t==e.size&&r.close()})}})}}catch(e){try{new Response("").body.getReader().read(),g=function(){return new Response(this).body}}catch(e){g=function(){throw new Error("Include https://github.com/MattiasBuelens/web-streams-polyfill")}}}}function w(e){return new Promise(function(t,r){e.onload=e.onerror=function(n){e.onload=e.onerror=null,"load"===n.type?t(e.result||e):r(new Error("Failed to read the blob/file"))}})}v.arrayBuffer||(v.arrayBuffer=function(){var e=new FileReader;return e.readAsArrayBuffer(this),w(e)}),v.text||(v.text=function(){var e=new FileReader;return e.readAsText(this),w(e)}),v.stream||(v.stream=g)},void 0===(o=n.apply(t,[t]))||(e.exports=o)},767(e,t,r){"use strict";!function(e){function t(){}function r(){}var n=String.fromCharCode,o={}.toString,i=o.call(e.SharedArrayBuffer),a=o(),s=e.Uint8Array,l=s||Array,c=s?ArrayBuffer:l,u=c.isView||function(e){return e&&"length"in e},f=o.call(c.prototype);c=r.prototype;var p=e.TextEncoder,d=new(s?Uint16Array:l)(32);t.prototype.decode=function(e){if(!u(e)){var t=o.call(e);if(t!==f&&t!==i&&t!==a)throw TypeError("Failed to execute 'decode' on 'TextDecoder': The provided value is not of type '(ArrayBuffer or ArrayBufferView)'");e=s?new l(e):e||[]}for(var r,c,p,h=t="",y=0,b=0|e.length,g=b-32|0,v=0,w=0,m=0,E=-1;y>4){case 15:if(2!=(p=255&e[y=y+1|0])>>6||247>6?w+4|0:24,c=c+256&768;case 13:case 12:v<<=6,v|=(31&c)<<6|63&(p=255&e[y=y+1|0]),w=w+7|0,y>6&&v>>w&&1114112>v?(c=v,0<=(v=v-65536|0)&&(E=55296+(v>>10)|0,c=56320+(1023&v)|0,31>m?(d[m]=E,m=m+1|0,E=-1):(p=E,E=c,c=p))):(y=y-(c>>=8)-1|0,c=65533),v=w=0,r=y<=g?32:b-y|0;default:d[m]=c;continue;case 11:case 10:case 9:case 8:}d[m]=65533}if(h+=n(d[0],d[1],d[2],d[3],d[4],d[5],d[6],d[7],d[8],d[9],d[10],d[11],d[12],d[13],d[14],d[15],d[16],d[17],d[18],d[19],d[20],d[21],d[22],d[23],d[24],d[25],d[26],d[27],d[28],d[29],d[30],d[31]),32>m&&(h=h.slice(0,m-32|0)),y>>31,E=-1,h.length=a)n[o]=a;else{if(2047>=a)n[o]=192|a>>6;else{e:{if(55296<=a)if(56319>=a){var c=0|e.charCodeAt(t=t+1|0);if(56320<=c&&57343>=c){if(65535<(a=(a<<10)+c-56613888|0)){n[o]=240|a>>18,n[o=o+1|0]=128|a>>12&63,n[o=o+1|0]=128|a>>6&63,n[o=o+1|0]=128|63&a;continue}break e}a=65533}else 57343>=a&&(a=65533);!i&&t<<1>12,n[o=o+1|0]=128|a>>6&63}n[o=o+1|0]=128|63&a}}return s?n.subarray(0,o):n.slice(0,o)},p||(e.TextDecoder=t,e.TextEncoder=r)}(""+void 0==typeof r.g?""+void 0==typeof self?this:self:r.g)}},t={};function r(n){var o=t[n];if(void 0!==o)return o.exports;var i=t[n]={exports:{}};return e[n].call(i.exports,i,i.exports,r),i.exports}r.g=function(){if("object"==typeof globalThis)return globalThis;try{return this||new Function("return this")()}catch(e){if("object"==typeof window)return window}}(),(()=>{"use strict";function e(e,r,...n){if(!e)throw new TypeError(t(r,n))}function t(e,t){let r=0;return e.replace(/%[os]/gu,()=>n(t[r++]))}function n(e){return"object"!=typeof e||null===e?String(e):Object.prototype.toString.call(e)}const o="undefined"!=typeof window?window:"undefined"!=typeof self?self:void 0!==r.g?r.g:"undefined"!=typeof globalThis?globalThis:void 0;let i;class a{constructor(e,t){this.code=e,this.message=t}warn(...e){var t;try{i;const r=(null!==(t=(new Error).stack)&&void 0!==t?t:"").replace(/^(?:.+?\n){2}/gu,"\n");console.warn(this.message,...e,r)}catch(e){}}}const s=new a("W01","Unable to initialize event under dispatching."),l=new a("W02","Assigning any falsy value to 'cancelBubble' property has no effect."),c=new a("W03","Assigning any truthy value to 'returnValue' property has no effect."),u=new a("W04","Unable to preventDefault on non-cancelable events."),f=new a("W05","Unable to preventDefault inside passive event listener invocation."),p=new a("W06","An event listener wasn't added because it has been added already: %o, %o"),d=new a("W07","The %o option value was abandoned because the event listener wasn't added as duplicated."),h=new a("W08","The 'callback' argument must be a function or an object that has 'handleEvent' method: %o");new a("W09","Event attribute handler must be a function: %o");class y{static get NONE(){return b}static get CAPTURING_PHASE(){return g}static get AT_TARGET(){return v}static get BUBBLING_PHASE(){return w}constructor(e,t){Object.defineProperty(this,"isTrusted",{value:!1,enumerable:!0});const r=null!=t?t:{};m.set(this,{type:String(e),bubbles:Boolean(r.bubbles),cancelable:Boolean(r.cancelable),composed:Boolean(r.composed),target:null,currentTarget:null,stopPropagationFlag:!1,stopImmediatePropagationFlag:!1,canceledFlag:!1,inPassiveListenerFlag:!1,dispatchFlag:!1,timeStamp:Date.now()})}get type(){return E(this).type}get target(){return E(this).target}get srcElement(){return E(this).target}get currentTarget(){return E(this).currentTarget}composedPath(){const e=E(this).currentTarget;return e?[e]:[]}get NONE(){return b}get CAPTURING_PHASE(){return g}get AT_TARGET(){return v}get BUBBLING_PHASE(){return w}get eventPhase(){return E(this).dispatchFlag?2:0}stopPropagation(){E(this).stopPropagationFlag=!0}get cancelBubble(){return E(this).stopPropagationFlag}set cancelBubble(e){e?E(this).stopPropagationFlag=!0:l.warn()}stopImmediatePropagation(){const e=E(this);e.stopPropagationFlag=e.stopImmediatePropagationFlag=!0}get bubbles(){return E(this).bubbles}get cancelable(){return E(this).cancelable}get returnValue(){return!E(this).canceledFlag}set returnValue(e){e?c.warn():A(E(this))}preventDefault(){A(E(this))}get defaultPrevented(){return E(this).canceledFlag}get composed(){return E(this).composed}get isTrusted(){return!1}get timeStamp(){return E(this).timeStamp}initEvent(e,t=!1,r=!1){const n=E(this);n.dispatchFlag?s.warn():m.set(this,{...n,type:String(e),bubbles:Boolean(t),cancelable:Boolean(r),target:null,currentTarget:null,stopPropagationFlag:!1,stopImmediatePropagationFlag:!1,canceledFlag:!1})}}const b=0,g=1,v=2,w=3,m=new WeakMap;function E(t,r="this"){const n=m.get(t);return e(null!=n,"'%s' must be an object that Event constructor created, but got another one: %o",r,t),n}function A(e){e.inPassiveListenerFlag?f.warn():e.cancelable?e.canceledFlag=!0:u.warn()}Object.defineProperty(y,"NONE",{enumerable:!0}),Object.defineProperty(y,"CAPTURING_PHASE",{enumerable:!0}),Object.defineProperty(y,"AT_TARGET",{enumerable:!0}),Object.defineProperty(y,"BUBBLING_PHASE",{enumerable:!0});const T=Object.getOwnPropertyNames(y.prototype);for(let e=0;eo,configurable:!0,enumerable:!0})}}class B extends y{static wrap(e){return new(L(e))(e)}constructor(e){super(e.type,{bubbles:e.bubbles,cancelable:e.cancelable,composed:e.composed}),e.cancelBubble&&super.stopPropagation(),e.defaultPrevented&&super.preventDefault(),P.set(this,{original:e});const t=Object.keys(e);for(let r=0;rr!==t),!1):(e.listeners.splice(t,1),!0)}j.set(Object.prototype,B),void 0!==o&&void 0!==o.Event&&j.set(o.Event.prototype,B);class H{constructor(){V.set(this,Object.create(null))}addEventListener(e,t,r){const n=q(this),{callback:o,capture:i,once:a,passive:s,signal:l,type:c}=function(e,t,r){var n;return G(t),"object"==typeof r&&null!==r?{type:String(e),callback:null!=t?t:void 0,capture:Boolean(r.capture),passive:Boolean(r.passive),once:Boolean(r.once),signal:null!==(n=r.signal)&&void 0!==n?n:void 0}:{type:String(e),callback:null!=t?t:void 0,capture:Boolean(r),passive:!1,once:!1,signal:void 0}}(e,t,r);if(null==o||(null==l?void 0:l.aborted))return;const u=function(e,t){var r;return null!==(r=e[t])&&void 0!==r?r:e[t]={attrCallback:void 0,attrListener:void 0,cow:!1,listeners:[]}}(n,c),f=C(u,o,i);-1===f?function(e,t,r,n,o,i){let a;i&&(a=N.bind(null,e,t,r),i.addEventListener("abort",a));const s=function(e,t,r,n,o,i){return{callback:e,flags:(t?1:0)|(r?2:0)|(n?4:0),signal:o,signalListener:i}}(t,r,n,o,i,a);e.cow?(e.cow=!1,e.listeners=[...e.listeners,s]):e.listeners.push(s)}(u,o,i,s,a,l):function(e,t,r,n){p.warn(D(e)?"capture":"bubble",e.callback),I(e)!==t&&d.warn("passive"),U(e)!==r&&d.warn("once"),e.signal!==n&&d.warn("signal")}(u.listeners[f],s,a,l)}removeEventListener(e,t,r){const n=q(this),{callback:o,capture:i,type:a}=function(e,t,r){return G(t),"object"==typeof r&&null!==r?{type:String(e),callback:null!=t?t:void 0,capture:Boolean(r.capture)}:{type:String(e),callback:null!=t?t:void 0,capture:Boolean(r)}}(e,t,r),s=n[a];null!=o&&s&&N(s,o,i)}dispatchEvent(e){const t=q(this)[String(e.type)];if(null==t)return!0;const r=e instanceof y?e:B.wrap(e),n=E(r,"event");if(n.dispatchFlag)throw i="This event has been in dispatching.",o.DOMException?new o.DOMException(i,"InvalidStateError"):(null==R&&(R=class e extends Error{constructor(t){super(t),Error.captureStackTrace&&Error.captureStackTrace(this,e)}get code(){return 11}get name(){return"InvalidStateError"}},Object.defineProperties(R.prototype,{code:{enumerable:!0},name:{enumerable:!0}}),_(R),_(R.prototype)),new R(i));var i;if(n.dispatchFlag=!0,n.target=n.currentTarget=this,!n.stopPropagationFlag){const{cow:e,listeners:o}=t;t.cow=!0;for(let i=0;i(e^this.getRandomValues(new Uint8Array(1))[0]&15>>e/4).toString(16))}},r(767);var z="undefined"!=typeof globalThis&&globalThis||"undefined"!=typeof self&&self||void 0!==r.g&&r.g||{},Y="URLSearchParams"in z,Z="Symbol"in z&&"iterator"in Symbol,K="FileReader"in z&&"Blob"in z&&function(){try{return new Blob,!0}catch(e){return!1}}(),Q="FormData"in z,$="ArrayBuffer"in z;if($)var J=["[object Int8Array]","[object Uint8Array]","[object Uint8ClampedArray]","[object Int16Array]","[object Uint16Array]","[object Int32Array]","[object Uint32Array]","[object Float32Array]","[object Float64Array]"],ee=ArrayBuffer.isView||function(e){return e&&J.indexOf(Object.prototype.toString.call(e))>-1};function te(e){if("string"!=typeof e&&(e=String(e)),/[^a-z0-9\-#$%&'*+.^_`|~!]/i.test(e)||""===e)throw new TypeError('Invalid character in header field name: "'+e+'"');return e.toLowerCase()}function re(e){return"string"!=typeof e&&(e=String(e)),e}function ne(e){var t={next:function(){var t=e.shift();return{done:void 0===t,value:t}}};return Z&&(t[Symbol.iterator]=function(){return t}),t}function oe(e){this.map={},e instanceof oe?e.forEach(function(e,t){this.append(t,e)},this):Array.isArray(e)?e.forEach(function(e){if(2!=e.length)throw new TypeError("Headers constructor: expected name/value pair to be length 2, found"+e.length);this.append(e[0],e[1])},this):e&&Object.getOwnPropertyNames(e).forEach(function(t){this.append(t,e[t])},this)}function ie(e){if(!e._noBody)return e.bodyUsed?Promise.reject(new TypeError("Already read")):void(e.bodyUsed=!0)}function ae(e){return new Promise(function(t,r){e.onload=function(){t(e.result)},e.onerror=function(){r(e.error)}})}function se(e){var t=new FileReader,r=ae(t);return t.readAsArrayBuffer(e),r}function le(e){if(e.slice)return e.slice(0);var t=new Uint8Array(e.byteLength);return t.set(new Uint8Array(e)),t.buffer}function ce(){return this.bodyUsed=!1,this._initBody=function(e){var t;this.bodyUsed=this.bodyUsed,this._bodyInit=e,e?"string"==typeof e?this._bodyText=e:K&&Blob.prototype.isPrototypeOf(e)?this._bodyBlob=e:Q&&FormData.prototype.isPrototypeOf(e)?this._bodyFormData=e:Y&&URLSearchParams.prototype.isPrototypeOf(e)?this._bodyText=e.toString():$&&K&&(t=e)&&DataView.prototype.isPrototypeOf(t)?(this._bodyArrayBuffer=le(e.buffer),this._bodyInit=new Blob([this._bodyArrayBuffer])):$&&(ArrayBuffer.prototype.isPrototypeOf(e)||ee(e))?this._bodyArrayBuffer=le(e):this._bodyText=e=Object.prototype.toString.call(e):(this._noBody=!0,this._bodyText=""),this.headers.get("content-type")||("string"==typeof e?this.headers.set("content-type","text/plain;charset=UTF-8"):this._bodyBlob&&this._bodyBlob.type?this.headers.set("content-type",this._bodyBlob.type):Y&&URLSearchParams.prototype.isPrototypeOf(e)&&this.headers.set("content-type","application/x-www-form-urlencoded;charset=UTF-8"))},K&&(this.blob=function(){var e=ie(this);if(e)return e;if(this._bodyBlob)return Promise.resolve(this._bodyBlob);if(this._bodyArrayBuffer)return Promise.resolve(new Blob([this._bodyArrayBuffer]));if(this._bodyFormData)throw new Error("could not read FormData body as blob");return Promise.resolve(new Blob([this._bodyText]))}),this.arrayBuffer=function(){if(this._bodyArrayBuffer)return ie(this)||(ArrayBuffer.isView(this._bodyArrayBuffer)?Promise.resolve(this._bodyArrayBuffer.buffer.slice(this._bodyArrayBuffer.byteOffset,this._bodyArrayBuffer.byteOffset+this._bodyArrayBuffer.byteLength)):Promise.resolve(this._bodyArrayBuffer));if(K)return this.blob().then(se);throw new Error("could not read as ArrayBuffer")},this.text=function(){var e,t,r,n,o,i=ie(this);if(i)return i;if(this._bodyBlob)return e=this._bodyBlob,r=ae(t=new FileReader),o=(n=/charset=([A-Za-z0-9_-]+)/.exec(e.type))?n[1]:"utf-8",t.readAsText(e,o),r;if(this._bodyArrayBuffer)return Promise.resolve(function(e){for(var t=new Uint8Array(e),r=new Array(t.length),n=0;n-1?n:r),this.mode=t.mode||this.mode||null,this.signal=t.signal||this.signal||function(){if("AbortController"in z)return(new AbortController).signal}(),this.referrer=null,this.redirect=t.redirect||"follow",("GET"===this.method||"HEAD"===this.method)&&o)throw new TypeError("Body not allowed for GET or HEAD requests");if(this._initBody(o),!("GET"!==this.method&&"HEAD"!==this.method||"no-store"!==t.cache&&"no-cache"!==t.cache)){var i=/([?&])_=[^&]*/;i.test(this.url)?this.url=this.url.replace(i,"$1_="+(new Date).getTime()):this.url+=(/\?/.test(this.url)?"&":"?")+"_="+(new Date).getTime()}}function pe(e){var t=new FormData;return e.trim().split("&").forEach(function(e){if(e){var r=e.split("="),n=r.shift().replace(/\+/g," "),o=r.join("=").replace(/\+/g," ");t.append(decodeURIComponent(n),decodeURIComponent(o))}}),t}function de(e,t){if(!(this instanceof de))throw new TypeError('Please use the "new" operator, this DOM object constructor cannot be called as a function.');if(t||(t={}),this.type="default",this.status=void 0===t.status?200:t.status,this.status<200||this.status>599)throw new RangeError("Failed to construct 'Response': The status provided (0) is outside the range [200, 599].");this.ok=this.status>=200&&this.status<300,this.statusText=void 0===t.statusText?"":""+t.statusText,this.headers=new oe(t.headers),this.url=t.url||"",this._initBody(e)}fe.prototype.clone=function(){return new fe(this,{body:this._bodyInit})},ce.call(fe.prototype),ce.call(de.prototype),de.prototype.clone=function(){return new de(this._bodyInit,{status:this.status,statusText:this.statusText,headers:new oe(this.headers),url:this.url})},de.error=function(){var e=new de(null,{status:200,statusText:""});return e.ok=!1,e.status=0,e.type="error",e};var he=[301,302,303,307,308];de.redirect=function(e,t){if(-1===he.indexOf(t))throw new RangeError("Invalid status code");return new de(null,{status:t,headers:{location:e}})};var ye=z.DOMException;try{new ye}catch(e){(ye=function(e,t){this.message=e,this.name=t;var r=Error(e);this.stack=r.stack}).prototype=Object.create(Error.prototype),ye.prototype.constructor=ye}function be(e,t){return new Promise(function(r,n){var o=new fe(e,t);if(o.signal&&o.signal.aborted)return n(new ye("Aborted","AbortError"));var i=new XMLHttpRequest;function a(){i.abort()}if(i.onload=function(){var e,t,n={statusText:i.statusText,headers:(e=i.getAllResponseHeaders()||"",t=new oe,e.replace(/\r?\n[\t ]+/g," ").split("\r").map(function(e){return 0===e.indexOf("\n")?e.substr(1,e.length):e}).forEach(function(e){var r=e.split(":"),n=r.shift().trim();if(n){var o=r.join(":").trim();try{t.append(n,o)}catch(e){console.warn("Response "+e.message)}}}),t)};0===o.url.indexOf("file://")&&(i.status<200||i.status>599)?n.status=200:n.status=i.status,n.url="responseURL"in i?i.responseURL:n.headers.get("X-Request-URL");var a="response"in i?i.response:i.responseText;setTimeout(function(){r(new de(a,n))},0)},i.onerror=function(){setTimeout(function(){n(new TypeError("Network request failed"))},0)},i.ontimeout=function(){setTimeout(function(){n(new TypeError("Network request timed out"))},0)},i.onabort=function(){setTimeout(function(){n(new ye("Aborted","AbortError"))},0)},i.open(o.method,function(e){try{return""===e&&z.location.href?z.location.href:e}catch(t){return e}}(o.url),!0),"include"===o.credentials?i.withCredentials=!0:"omit"===o.credentials&&(i.withCredentials=!1),"responseType"in i&&(K?i.responseType="blob":$&&(i.responseType="arraybuffer")),"redirect"in i&&(i.redirect=o.redirect),t&&"object"==typeof t.headers&&!(t.headers instanceof oe||z.Headers&&t.headers instanceof z.Headers)){var s=[];Object.getOwnPropertyNames(t.headers).forEach(function(e){s.push(te(e)),i.setRequestHeader(e,re(t.headers[e]))}),o.headers.forEach(function(e,t){-1===s.indexOf(t)&&i.setRequestHeader(t,e)})}else o.headers.forEach(function(e,t){i.setRequestHeader(t,e)});o.signal&&(o.signal.addEventListener("abort",a),i.onreadystatechange=function(){4===i.readyState&&o.signal.removeEventListener("abort",a)}),i.send(void 0===o._bodyInit?null:o._bodyInit)})}be.polyfill=!0,z.fetch||(z.fetch=be,z.Headers=oe,z.Request=fe,z.Response=de)})()})(); \ No newline at end of file +(()=>{var e={847(e,t,n){var r,o;r=function(){"use strict";function e(e,t){(null==t||t>e.length)&&(t=e.length);for(var n=0,r=Array(t);n=t.length?{done:!0}:{done:!1,value:t[o++]}},e:function(e){throw e},f:a}}throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.")}var i,l=!0,s=!1;return{s:function(){r=r.call(t)},n:function(){var e=r.next();return l=e.done,e},e:function(e){s=!0,i=e},f:function(){try{l||null==r.return||r.return()}finally{if(s)throw i}}}}function l(){return l="undefined"!=typeof Reflect&&Reflect.get?Reflect.get.bind():function(e,t,n){var r=function(e,t){for(;!{}.hasOwnProperty.call(e,t)&&null!==(e=s(e)););return e}(e,t);if(r){var o=Object.getOwnPropertyDescriptor(r,t);return o.get?o.get.call(arguments.length<3?e:n):o.value}},l.apply(null,arguments)}function s(e){return s=Object.setPrototypeOf?Object.getPrototypeOf.bind():function(e){return e.__proto__||Object.getPrototypeOf(e)},s(e)}function c(){try{var e=!Boolean.prototype.valueOf.call(Reflect.construct(Boolean,[],function(){}))}catch(e){}return(c=function(){return!!e})()}function u(e,t){return u=Object.setPrototypeOf?Object.setPrototypeOf.bind():function(e,t){return e.__proto__=t,e},u(e,t)}function f(e){var t=function(e,t){if("object"!=typeof e||!e)return e;var n=e[Symbol.toPrimitive];if(void 0!==n){var r=n.call(e,t||"default");if("object"!=typeof r)return r;throw new TypeError("@@toPrimitive must return a primitive value.")}return("string"===t?String:Number)(e)}(e,"string");return"symbol"==typeof t?t:t+""}!function(e){e.AbortSignal,e.AbortController}("undefined"!=typeof self?self:n.g);var p=function(){return a(function e(){r(this,e),Object.defineProperty(this,"listeners",{value:{},writable:!0,configurable:!0})},[{key:"addEventListener",value:function(e,t,n){e in this.listeners||(this.listeners[e]=[]),this.listeners[e].push({callback:t,options:n})}},{key:"removeEventListener",value:function(e,t){if(e in this.listeners)for(var n=this.listeners[e],r=0,o=n.length;r>1)+7),l=new r(i>>3<<3);t=55296&&s<=56319){if(t=55296&&s<=56319)continue}if(o+4>l.length){i+=8,i=(i*=1+t/e.length*2)>>3<<3;var u=new Uint8Array(i);u.set(l),l=u}if(4294967168&s){if(4294965248&s)if(4294901760&s){if(4292870144&s)continue;l[o++]=s>>18&7|240,l[o++]=s>>12&63|128,l[o++]=s>>6&63|128}else l[o++]=s>>12&15|224,l[o++]=s>>6&63|128;else l[o++]=s>>6&31|192;l[o++]=63&s|128}else l[o++]=s}return l.slice(0,o)},y="function"==typeof TextDecoder?TextDecoder.prototype.decode.bind(new TextDecoder):function(e){for(var t=e.length,n=[],r=0;r239?4:s>223?3:s>191?2:1;if(r+u<=t)switch(u){case 1:s<128&&(c=s);break;case 2:128==(192&(o=e[r+1]))&&(l=(31&s)<<6|63&o)>127&&(c=l);break;case 3:o=e[r+1],a=e[r+2],128==(192&o)&&128==(192&a)&&(l=(15&s)<<12|(63&o)<<6|63&a)>2047&&(l<55296||l>57343)&&(c=l);break;case 4:o=e[r+1],a=e[r+2],i=e[r+3],128==(192&o)&&128==(192&a)&&128==(192&i)&&(l=(15&s)<<18|(63&o)<<12|(63&a)<<6|63&i)>65535&&l<1114112&&(c=l)}null===c?(c=65533,u=1):c>65535&&(c-=65536,n.push(c>>>10&1023|55296),c=56320|1023&c),n.push(c),r+=u}for(var f=n.length,p="",b=0;b>2,u=(3&o)<<4|i>>4,f=(15&i)<<2|s>>6,p=63&s;l||(p=64,a||(f=64)),n.push(t[c],t[u],t[f],t[p])}return n.join("")}var l=Object.create||function(e){function t(){}return t.prototype=e,new t};function s(e){return Object.prototype.toString.call(e).slice(8,-1)}function c(e,t){return"object"==typeof e&&Object.prototype.isPrototypeOf.call(e.prototype,t)}function u(e){return"DataView"===s(e)||c(a.DataView,e)}var f=["Int8Array","Uint8Array","Uint8ClampedArray","Int16Array","Uint16Array","Int32Array","Uint32Array","Float32Array","Float64Array","ArrayBuffer"];function p(e){return t=f,n=s(e),-1!==t.indexOf(n)||c(a.ArrayBuffer,e);var t,n}function b(e,n){n=null==n?{}:n;for(var r=0,o=(e=e?e.slice():[]).length;r=t.size&&n.close()})}})}}catch(e){try{new ReadableStream({}),h=function(e){var t=0;return new ReadableStream({pull:function(n){return e.slice(t,t+524288).arrayBuffer().then(function(r){t+=r.byteLength;var o=new Uint8Array(r);n.enqueue(o),t==e.size&&n.close()})}})}}catch(e){try{new Response("").body.getReader().read(),h=function(){return new Response(this).body}}catch(e){h=function(){throw new Error("Include https://github.com/MattiasBuelens/web-streams-polyfill")}}}}function w(e){return new Promise(function(t,n){e.onload=e.onerror=function(r){e.onload=e.onerror=null,"load"===r.type?t(e.result||e):n(new Error("Failed to read the blob/file"))}})}v.arrayBuffer||(v.arrayBuffer=function(){var e=new FileReader;return e.readAsArrayBuffer(this),w(e)}),v.text||(v.text=function(){var e=new FileReader;return e.readAsText(this),w(e)}),v.stream||(v.stream=h)},void 0===(o=r.apply(t,[t]))||(e.exports=o)},767(e,t,n){"use strict";!function(e){function t(){}function n(){}var r=String.fromCharCode,o={}.toString,a=o.call(e.SharedArrayBuffer),i=o(),l=e.Uint8Array,s=l||Array,c=l?ArrayBuffer:s,u=c.isView||function(e){return e&&"length"in e},f=o.call(c.prototype);c=n.prototype;var p=e.TextEncoder,b=new(l?Uint16Array:s)(32);t.prototype.decode=function(e){if(!u(e)){var t=o.call(e);if(t!==f&&t!==a&&t!==i)throw TypeError("Failed to execute 'decode' on 'TextDecoder': The provided value is not of type '(ArrayBuffer or ArrayBufferView)'");e=l?new s(e):e||[]}for(var n,c,p,d=t="",y=0,g=0|e.length,h=g-32|0,v=0,w=0,E=0,m=-1;y>4){case 15:if(2!=(p=255&e[y=y+1|0])>>6||247>6?w+4|0:24,c=c+256&768;case 13:case 12:v<<=6,v|=(31&c)<<6|63&(p=255&e[y=y+1|0]),w=w+7|0,y>6&&v>>w&&1114112>v?(c=v,0<=(v=v-65536|0)&&(m=55296+(v>>10)|0,c=56320+(1023&v)|0,31>E?(b[E]=m,E=E+1|0,m=-1):(p=m,m=c,c=p))):(y=y-(c>>=8)-1|0,c=65533),v=w=0,n=y<=h?32:g-y|0;default:b[E]=c;continue;case 11:case 10:case 9:case 8:}b[E]=65533}if(d+=r(b[0],b[1],b[2],b[3],b[4],b[5],b[6],b[7],b[8],b[9],b[10],b[11],b[12],b[13],b[14],b[15],b[16],b[17],b[18],b[19],b[20],b[21],b[22],b[23],b[24],b[25],b[26],b[27],b[28],b[29],b[30],b[31]),32>E&&(d=d.slice(0,E-32|0)),y>>31,m=-1,d.length=i)r[o]=i;else{if(2047>=i)r[o]=192|i>>6;else{e:{if(55296<=i)if(56319>=i){var c=0|e.charCodeAt(t=t+1|0);if(56320<=c&&57343>=c){if(65535<(i=(i<<10)+c-56613888|0)){r[o]=240|i>>18,r[o=o+1|0]=128|i>>12&63,r[o=o+1|0]=128|i>>6&63,r[o=o+1|0]=128|63&i;continue}break e}i=65533}else 57343>=i&&(i=65533);!a&&t<<1>12,r[o=o+1|0]=128|i>>6&63}r[o=o+1|0]=128|63&i}}return l?r.subarray(0,o):r.slice(0,o)},p||(e.TextDecoder=t,e.TextEncoder=n)}(""+void 0==typeof n.g?""+void 0==typeof self?this:self:n.g)}},t={};function n(r){var o=t[r];if(void 0!==o)return o.exports;var a=t[r]={exports:{}};return e[r].call(a.exports,a,a.exports,n),a.exports}n.g=function(){if("object"==typeof globalThis)return globalThis;try{return this||new Function("return this")()}catch(e){if("object"==typeof window)return window}}(),(()=>{"use strict";function e(e,n,...r){if(!e)throw new TypeError(t(n,r))}function t(e,t){let n=0;return e.replace(/%[os]/gu,()=>r(t[n++]))}function r(e){return"object"!=typeof e||null===e?String(e):Object.prototype.toString.call(e)}const o="undefined"!=typeof window?window:"undefined"!=typeof self?self:void 0!==n.g?n.g:"undefined"!=typeof globalThis?globalThis:void 0;let a;class i{constructor(e,t){this.code=e,this.message=t}warn(...e){var t;try{a;const n=(null!==(t=(new Error).stack)&&void 0!==t?t:"").replace(/^(?:.+?\n){2}/gu,"\n");console.warn(this.message,...e,n)}catch(e){}}}const l=new i("W01","Unable to initialize event under dispatching."),s=new i("W02","Assigning any falsy value to 'cancelBubble' property has no effect."),c=new i("W03","Assigning any truthy value to 'returnValue' property has no effect."),u=new i("W04","Unable to preventDefault on non-cancelable events."),f=new i("W05","Unable to preventDefault inside passive event listener invocation."),p=new i("W06","An event listener wasn't added because it has been added already: %o, %o"),b=new i("W07","The %o option value was abandoned because the event listener wasn't added as duplicated."),d=new i("W08","The 'callback' argument must be a function or an object that has 'handleEvent' method: %o");new i("W09","Event attribute handler must be a function: %o");class y{static get NONE(){return g}static get CAPTURING_PHASE(){return h}static get AT_TARGET(){return v}static get BUBBLING_PHASE(){return w}constructor(e,t){Object.defineProperty(this,"isTrusted",{value:!1,enumerable:!0});const n=null!=t?t:{};E.set(this,{type:String(e),bubbles:Boolean(n.bubbles),cancelable:Boolean(n.cancelable),composed:Boolean(n.composed),target:null,currentTarget:null,stopPropagationFlag:!1,stopImmediatePropagationFlag:!1,canceledFlag:!1,inPassiveListenerFlag:!1,dispatchFlag:!1,timeStamp:Date.now()})}get type(){return m(this).type}get target(){return m(this).target}get srcElement(){return m(this).target}get currentTarget(){return m(this).currentTarget}composedPath(){const e=m(this).currentTarget;return e?[e]:[]}get NONE(){return g}get CAPTURING_PHASE(){return h}get AT_TARGET(){return v}get BUBBLING_PHASE(){return w}get eventPhase(){return m(this).dispatchFlag?2:0}stopPropagation(){m(this).stopPropagationFlag=!0}get cancelBubble(){return m(this).stopPropagationFlag}set cancelBubble(e){e?m(this).stopPropagationFlag=!0:s.warn()}stopImmediatePropagation(){const e=m(this);e.stopPropagationFlag=e.stopImmediatePropagationFlag=!0}get bubbles(){return m(this).bubbles}get cancelable(){return m(this).cancelable}get returnValue(){return!m(this).canceledFlag}set returnValue(e){e?c.warn():R(m(this))}preventDefault(){R(m(this))}get defaultPrevented(){return m(this).canceledFlag}get composed(){return m(this).composed}get isTrusted(){return!1}get timeStamp(){return m(this).timeStamp}initEvent(e,t=!1,n=!1){const r=m(this);r.dispatchFlag?l.warn():E.set(this,{...r,type:String(e),bubbles:Boolean(t),cancelable:Boolean(n),target:null,currentTarget:null,stopPropagationFlag:!1,stopImmediatePropagationFlag:!1,canceledFlag:!1})}}const g=0,h=1,v=2,w=3,E=new WeakMap;function m(t,n="this"){const r=E.get(t);return e(null!=r,"'%s' must be an object that Event constructor created, but got another one: %o",n,t),r}function R(e){e.inPassiveListenerFlag?f.warn():e.cancelable?e.canceledFlag=!0:u.warn()}Object.defineProperty(y,"NONE",{enumerable:!0}),Object.defineProperty(y,"CAPTURING_PHASE",{enumerable:!0}),Object.defineProperty(y,"AT_TARGET",{enumerable:!0}),Object.defineProperty(y,"BUBBLING_PHASE",{enumerable:!0});const A=Object.getOwnPropertyNames(y.prototype);for(let e=0;eo,configurable:!0,enumerable:!0})}}class _ extends y{static wrap(e){return new(j(e))(e)}constructor(e){super(e.type,{bubbles:e.bubbles,cancelable:e.cancelable,composed:e.composed}),e.cancelBubble&&super.stopPropagation(),e.defaultPrevented&&super.preventDefault(),S.set(this,{original:e});const t=Object.keys(e);for(let n=0;nn!==t),!1):(e.listeners.splice(t,1),!0)}L.set(Object.prototype,_),void 0!==o&&void 0!==o.Event&&L.set(o.Event.prototype,_);class V{constructor(){H.set(this,Object.create(null))}addEventListener(e,t,n){const r=W(this),{callback:o,capture:a,once:i,passive:l,signal:s,type:c}=function(e,t,n){var r;return q(t),"object"==typeof n&&null!==n?{type:String(e),callback:null!=t?t:void 0,capture:Boolean(n.capture),passive:Boolean(n.passive),once:Boolean(n.once),signal:null!==(r=n.signal)&&void 0!==r?r:void 0}:{type:String(e),callback:null!=t?t:void 0,capture:Boolean(n),passive:!1,once:!1,signal:void 0}}(e,t,n);if(null==o||(null==s?void 0:s.aborted))return;const u=function(e,t){var n;return null!==(n=e[t])&&void 0!==n?n:e[t]={attrCallback:void 0,attrListener:void 0,cow:!1,listeners:[]}}(r,c),f=N(u,o,a);-1===f?function(e,t,n,r,o,a){let i;a&&(i=C.bind(null,e,t,n),a.addEventListener("abort",i));const l=function(e,t,n,r,o,a){return{callback:e,flags:(t?1:0)|(n?2:0)|(r?4:0),signal:o,signalListener:a}}(t,n,r,o,a,i);e.cow?(e.cow=!1,e.listeners=[...e.listeners,l]):e.listeners.push(l)}(u,o,a,l,i,s):function(e,t,n,r){p.warn(D(e)?"capture":"bubble",e.callback),I(e)!==t&&b.warn("passive"),k(e)!==n&&b.warn("once"),e.signal!==r&&b.warn("signal")}(u.listeners[f],l,i,s)}removeEventListener(e,t,n){const r=W(this),{callback:o,capture:a,type:i}=function(e,t,n){return q(t),"object"==typeof n&&null!==n?{type:String(e),callback:null!=t?t:void 0,capture:Boolean(n.capture)}:{type:String(e),callback:null!=t?t:void 0,capture:Boolean(n)}}(e,t,n),l=r[i];null!=o&&l&&C(l,o,a)}dispatchEvent(e){const t=W(this)[String(e.type)];if(null==t)return!0;const n=e instanceof y?e:_.wrap(e),r=m(n,"event");if(r.dispatchFlag)throw a="This event has been in dispatching.",o.DOMException?new o.DOMException(a,"InvalidStateError"):(null==O&&(O=class e extends Error{constructor(t){super(t),Error.captureStackTrace&&Error.captureStackTrace(this,e)}get code(){return 11}get name(){return"InvalidStateError"}},Object.defineProperties(O.prototype,{code:{enumerable:!0},name:{enumerable:!0}}),P(O),P(O.prototype)),new O(a));var a;if(r.dispatchFlag=!0,r.target=r.currentTarget=this,!r.stopPropagationFlag){const{cow:e,listeners:o}=t;t.cow=!0;for(let a=0;a(e^this.getRandomValues(new Uint8Array(1))[0]&15>>e/4).toString(16))}},n(767)})()})(); \ No newline at end of file diff --git a/pkg/download/engine/polyfill/package-lock.json b/pkg/download/engine/polyfill/package-lock.json index a1ebe8d8..fdb12da0 100644 --- a/pkg/download/engine/polyfill/package-lock.json +++ b/pkg/download/engine/polyfill/package-lock.json @@ -13,8 +13,7 @@ "abortcontroller-polyfill": "^1.7.8", "blob-polyfill": "^7.0.20220408", "event-target-shim": "^6.0.2", - "fastestsmallesttextencoderdecoder": "^1.0.22", - "whatwg-fetch": "^3.6.20" + "fastestsmallesttextencoderdecoder": "^1.0.22" }, "devDependencies": { "patch-package": "^8.0.0", @@ -2106,12 +2105,6 @@ "node": ">=10.13.0" } }, - "node_modules/whatwg-fetch": { - "version": "3.6.20", - "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.6.20.tgz", - "integrity": "sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg==", - "license": "MIT" - }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", diff --git a/pkg/download/engine/polyfill/package.json b/pkg/download/engine/polyfill/package.json index a49faa56..21a16759 100644 --- a/pkg/download/engine/polyfill/package.json +++ b/pkg/download/engine/polyfill/package.json @@ -21,7 +21,6 @@ "abortcontroller-polyfill": "^1.7.8", "blob-polyfill": "^7.0.20220408", "event-target-shim": "^6.0.2", - "fastestsmallesttextencoderdecoder": "^1.0.22", - "whatwg-fetch": "^3.6.20" + "fastestsmallesttextencoderdecoder": "^1.0.22" } } diff --git a/pkg/download/engine/polyfill/patches/whatwg-fetch+3.6.20.patch b/pkg/download/engine/polyfill/patches/whatwg-fetch+3.6.20.patch deleted file mode 100644 index d5faf51a..00000000 --- a/pkg/download/engine/polyfill/patches/whatwg-fetch+3.6.20.patch +++ /dev/null @@ -1,46 +0,0 @@ -diff --git a/node_modules/whatwg-fetch/dist/fetch.umd.js b/node_modules/whatwg-fetch/dist/fetch.umd.js -index 7a0d852..604691e 100644 ---- a/node_modules/whatwg-fetch/dist/fetch.umd.js -+++ b/node_modules/whatwg-fetch/dist/fetch.umd.js -@@ -394,6 +394,7 @@ - } - }()); - this.referrer = null; -+ this.redirect = options.redirect || 'follow' - - if ((this.method === 'GET' || this.method === 'HEAD') && body) { - throw new TypeError('Body not allowed for GET or HEAD requests') -@@ -606,6 +607,10 @@ - } - } - -+ if ('redirect' in xhr) { -+ xhr.redirect = request.redirect -+ } -+ - if (init && typeof init.headers === 'object' && !(init.headers instanceof Headers || (g.Headers && init.headers instanceof g.Headers))) { - var names = []; - Object.getOwnPropertyNames(init.headers).forEach(function(name) { -diff --git a/node_modules/whatwg-fetch/fetch.js b/node_modules/whatwg-fetch/fetch.js -index f39a983..d1fc903 100644 ---- a/node_modules/whatwg-fetch/fetch.js -+++ b/node_modules/whatwg-fetch/fetch.js -@@ -388,6 +388,7 @@ export function Request(input, options) { - } - }()); - this.referrer = null -+ this.redirect = options.redirect || 'follow' - - if ((this.method === 'GET' || this.method === 'HEAD') && body) { - throw new TypeError('Body not allowed for GET or HEAD requests') -@@ -600,6 +601,10 @@ export function fetch(input, init) { - } - } - -+ if ('redirect' in xhr) { -+ xhr.redirect = request.redirect -+ } -+ - if (init && typeof init.headers === 'object' && !(init.headers instanceof Headers || (g.Headers && init.headers instanceof g.Headers))) { - var names = []; - Object.getOwnPropertyNames(init.headers).forEach(function(name) { diff --git a/pkg/download/engine/polyfill/src/fetch/index.js b/pkg/download/engine/polyfill/src/fetch/index.js deleted file mode 100644 index 35dae088..00000000 --- a/pkg/download/engine/polyfill/src/fetch/index.js +++ /dev/null @@ -1 +0,0 @@ -import 'whatwg-fetch' \ No newline at end of file diff --git a/pkg/download/engine/polyfill/src/index.js b/pkg/download/engine/polyfill/src/index.js index b989f17b..4579875f 100644 --- a/pkg/download/engine/polyfill/src/index.js +++ b/pkg/download/engine/polyfill/src/index.js @@ -4,4 +4,3 @@ import "./blob/index.js" import "./crypto/index.js" // polyfill TextEncoder import 'fastestsmallesttextencoderdecoder'; -import "./fetch/index.js" diff --git a/pkg/download/engine/testdata/wpt/LICENSE.md b/pkg/download/engine/testdata/wpt/LICENSE.md new file mode 100644 index 00000000..39c46d03 --- /dev/null +++ b/pkg/download/engine/testdata/wpt/LICENSE.md @@ -0,0 +1,11 @@ +# The 3-Clause BSD License + +Copyright © web-platform-tests contributors + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/pkg/download/engine/testdata/wpt/UPSTREAM.md b/pkg/download/engine/testdata/wpt/UPSTREAM.md new file mode 100644 index 00000000..e858ecb9 --- /dev/null +++ b/pkg/download/engine/testdata/wpt/UPSTREAM.md @@ -0,0 +1,28 @@ +# Web Platform Tests provenance + +The files under `fetch/` are copied from the official +[web-platform-tests/wpt](https://github.com/web-platform-tests/wpt) repository. + +- Upstream commit: `ce5d9e7e28b27528213bceea40d9e78462487105` +- License: BSD-3-Clause; see `LICENSE.md` +- Canonical Fetch tests: `fetch/` + +Gopeed executes applicable worker-style Fetch API tests at the extension +JavaScript boundary. The checked-in runner currently executes unchanged WPT +tests for Headers, no-CORS header guards, Request construction and body +consumption, ReadableStream bodies, keepalive, Request errors, Response +construction and body consumption, immutable network response headers, and +the Response static constructors. + +Passing this subset is an extension-worker Fetch API compatibility claim, not +a claim that Gopeed is a browser. Browser-only suites that require a document, +CORS origin enforcement or preflight, CSP, Mixed Content, Service Workers, +browser HTTP cache, navigation, browser authentication UI, or WPT's +multi-origin server infrastructure are intentionally excluded. Gopeed also +keeps one extension-specific behavior: `redirect: "manual"` exposes the HTTP +redirect response so download extensions can inspect `Location`, instead of +returning a browser `opaqueredirect` response. + +`wpt_harness.js` is a small Goja adapter maintained by Gopeed; it is not copied +from WPT. Test logic below `fetch/` is unchanged; the checked-in copy may only +normalize a final newline. diff --git a/pkg/download/engine/testdata/wpt/fetch/api/cors/resources/not-cors-safelisted.json b/pkg/download/engine/testdata/wpt/fetch/api/cors/resources/not-cors-safelisted.json new file mode 100644 index 00000000..945dc0f9 --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/cors/resources/not-cors-safelisted.json @@ -0,0 +1,13 @@ +[ + ["accept", "\""], + ["accept", "012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678"], + ["accept-language", "\u0001"], + ["accept-language", "@"], + ["authorization", "basics"], + ["content-language", "\u0001"], + ["content-language", "@"], + ["content-type", "text/html"], + ["content-type", "text/plain; long=0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901"], + ["range", "bytes 0-"], + ["test", "hi"] +] diff --git a/pkg/download/engine/testdata/wpt/fetch/api/headers/header-setcookie.any.js b/pkg/download/engine/testdata/wpt/fetch/api/headers/header-setcookie.any.js new file mode 100644 index 00000000..cafb780c --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/headers/header-setcookie.any.js @@ -0,0 +1,266 @@ +// META: title=Headers set-cookie special cases +// META: global=window,worker + +const headerList = [ + ["set-cookie", "foo=bar"], + ["Set-Cookie", "fizz=buzz; domain=example.com"], +]; + +const setCookie2HeaderList = [ + ["set-cookie2", "foo2=bar2"], + ["Set-Cookie2", "fizz2=buzz2; domain=example2.com"], +]; + +function assert_nested_array_equals(actual, expected) { + assert_equals(actual.length, expected.length, "Array length is not equal"); + for (let i = 0; i < expected.length; i++) { + assert_array_equals(actual[i], expected[i]); + } +} + +test(function () { + const headers = new Headers(headerList); + assert_equals( + headers.get("set-cookie"), + "foo=bar, fizz=buzz; domain=example.com", + ); +}, "Headers.prototype.get combines set-cookie headers in order"); + +test(function () { + const headers = new Headers(headerList); + const list = [...headers]; + assert_nested_array_equals(list, [ + ["set-cookie", "foo=bar"], + ["set-cookie", "fizz=buzz; domain=example.com"], + ]); +}, "Headers iterator does not combine set-cookie headers"); + +test(function () { + const headers = new Headers(setCookie2HeaderList); + const list = [...headers]; + assert_nested_array_equals(list, [ + ["set-cookie2", "foo2=bar2, fizz2=buzz2; domain=example2.com"], + ]); +}, "Headers iterator does not special case set-cookie2 headers"); + +test(function () { + const headers = new Headers([...headerList, ...setCookie2HeaderList]); + const list = [...headers]; + assert_nested_array_equals(list, [ + ["set-cookie", "foo=bar"], + ["set-cookie", "fizz=buzz; domain=example.com"], + ["set-cookie2", "foo2=bar2, fizz2=buzz2; domain=example2.com"], + ]); +}, "Headers iterator does not combine set-cookie & set-cookie2 headers"); + +test(function () { + // Values are in non alphabetic order, and the iterator should yield in the + // headers in the exact order of the input. + const headers = new Headers([ + ["set-cookie", "z=z"], + ["set-cookie", "a=a"], + ["set-cookie", "n=n"], + ]); + const list = [...headers]; + assert_nested_array_equals(list, [ + ["set-cookie", "z=z"], + ["set-cookie", "a=a"], + ["set-cookie", "n=n"], + ]); +}, "Headers iterator preserves set-cookie ordering"); + +test( + function () { + const headers = new Headers([ + ["xylophone-header", "1"], + ["best-header", "2"], + ["set-cookie", "3"], + ["a-cool-header", "4"], + ["set-cookie", "5"], + ["a-cool-header", "6"], + ["best-header", "7"], + ]); + const list = [...headers]; + assert_nested_array_equals(list, [ + ["a-cool-header", "4, 6"], + ["best-header", "2, 7"], + ["set-cookie", "3"], + ["set-cookie", "5"], + ["xylophone-header", "1"], + ]); + }, + "Headers iterator preserves per header ordering, but sorts keys alphabetically", +); + +test( + function () { + const headers = new Headers([ + ["xylophone-header", "7"], + ["best-header", "6"], + ["set-cookie", "5"], + ["a-cool-header", "4"], + ["set-cookie", "3"], + ["a-cool-header", "2"], + ["best-header", "1"], + ]); + const list = [...headers]; + assert_nested_array_equals(list, [ + ["a-cool-header", "4, 2"], + ["best-header", "6, 1"], + ["set-cookie", "5"], + ["set-cookie", "3"], + ["xylophone-header", "7"], + ]); + }, + "Headers iterator preserves per header ordering, but sorts keys alphabetically (and ignores value ordering)", +); + +test(function () { + const headers = new Headers([["fizz", "buzz"], ["X-Header", "test"]]); + const iterator = headers[Symbol.iterator](); + assert_array_equals(iterator.next().value, ["fizz", "buzz"]); + headers.append("Set-Cookie", "a=b"); + assert_array_equals(iterator.next().value, ["set-cookie", "a=b"]); + headers.append("Accept", "text/html"); + assert_array_equals(iterator.next().value, ["set-cookie", "a=b"]); + assert_array_equals(iterator.next().value, ["x-header", "test"]); + headers.append("set-cookie", "c=d"); + assert_array_equals(iterator.next().value, ["x-header", "test"]); + assert_true(iterator.next().done); +}, "Headers iterator is correctly updated with set-cookie changes"); + +test(function () { + const headers = new Headers([ + ["set-cookie", "a"], + ["set-cookie", "b"], + ["set-cookie", "c"] + ]); + const iterator = headers[Symbol.iterator](); + assert_array_equals(iterator.next().value, ["set-cookie", "a"]); + headers.delete("set-cookie"); + headers.append("set-cookie", "d"); + headers.append("set-cookie", "e"); + headers.append("set-cookie", "f"); + assert_array_equals(iterator.next().value, ["set-cookie", "e"]); + assert_array_equals(iterator.next().value, ["set-cookie", "f"]); + assert_true(iterator.next().done); +}, "Headers iterator is correctly updated with set-cookie changes #2"); + +test(function () { + const headers = new Headers(headerList); + assert_true(headers.has("sEt-cOoKiE")); +}, "Headers.prototype.has works for set-cookie"); + +test(function () { + const headers = new Headers(setCookie2HeaderList); + headers.append("set-Cookie", "foo=bar"); + headers.append("sEt-cOoKiE", "fizz=buzz"); + const list = [...headers]; + assert_nested_array_equals(list, [ + ["set-cookie", "foo=bar"], + ["set-cookie", "fizz=buzz"], + ["set-cookie2", "foo2=bar2, fizz2=buzz2; domain=example2.com"], + ]); +}, "Headers.prototype.append works for set-cookie"); + +test(function () { + const headers = new Headers(headerList); + headers.set("set-cookie", "foo2=bar2"); + const list = [...headers]; + assert_nested_array_equals(list, [ + ["set-cookie", "foo2=bar2"], + ]); +}, "Headers.prototype.set works for set-cookie"); + +test(function () { + const headers = new Headers(headerList); + headers.delete("set-Cookie"); + const list = [...headers]; + assert_nested_array_equals(list, []); +}, "Headers.prototype.delete works for set-cookie"); + +test(function () { + const headers = new Headers(); + assert_array_equals(headers.getSetCookie(), []); +}, "Headers.prototype.getSetCookie with no headers present"); + +test(function () { + const headers = new Headers([headerList[0]]); + assert_array_equals(headers.getSetCookie(), ["foo=bar"]); +}, "Headers.prototype.getSetCookie with one header"); + +test(function () { + const headers = new Headers({ "Set-Cookie": "foo=bar" }); + assert_array_equals(headers.getSetCookie(), ["foo=bar"]); +}, "Headers.prototype.getSetCookie with one header created from an object"); + +test(function () { + const headers = new Headers(headerList); + assert_array_equals(headers.getSetCookie(), [ + "foo=bar", + "fizz=buzz; domain=example.com", + ]); +}, "Headers.prototype.getSetCookie with multiple headers"); + +test(function () { + const headers = new Headers([["set-cookie", ""]]); + assert_array_equals(headers.getSetCookie(), [""]); +}, "Headers.prototype.getSetCookie with an empty header"); + +test(function () { + const headers = new Headers([["set-cookie", "x"], ["set-cookie", "x"]]); + assert_array_equals(headers.getSetCookie(), ["x", "x"]); +}, "Headers.prototype.getSetCookie with two equal headers"); + +test(function () { + const headers = new Headers([ + ["set-cookie2", "x"], + ["set-cookie", "y"], + ["set-cookie2", "z"], + ]); + assert_array_equals(headers.getSetCookie(), ["y"]); +}, "Headers.prototype.getSetCookie ignores set-cookie2 headers"); + +test(function () { + // Values are in non alphabetic order, and the iterator should yield in the + // headers in the exact order of the input. + const headers = new Headers([ + ["set-cookie", "z=z"], + ["set-cookie", "a=a"], + ["set-cookie", "n=n"], + ]); + assert_array_equals(headers.getSetCookie(), ["z=z", "a=a", "n=n"]); +}, "Headers.prototype.getSetCookie preserves header ordering"); + +test(function () { + const headers = new Headers({"Set-Cookie": " a=b\n"}); + headers.append("set-cookie", "\n\rc=d "); + assert_nested_array_equals([...headers], [ + ["set-cookie", "a=b"], + ["set-cookie", "c=d"] + ]); + headers.set("set-cookie", "\te=f "); + assert_nested_array_equals([...headers], [["set-cookie", "e=f"]]); +}, "Adding Set-Cookie headers normalizes their value"); + +test(function () { + assert_throws_js(TypeError, () => { + new Headers({"set-cookie": "\0"}); + }); + + const headers = new Headers(); + assert_throws_js(TypeError, () => { + headers.append("Set-Cookie", "a\nb"); + }); + assert_throws_js(TypeError, () => { + headers.set("Set-Cookie", "a\rb"); + }); +}, "Adding invalid Set-Cookie headers throws"); + +test(function () { + const response = new Response(); + response.headers.append("Set-Cookie", "foo=bar"); + assert_array_equals(response.headers.getSetCookie(), []); + response.headers.append("sEt-cOokIe", "bar=baz"); + assert_array_equals(response.headers.getSetCookie(), []); +}, "Set-Cookie is a forbidden response header"); diff --git a/pkg/download/engine/testdata/wpt/fetch/api/headers/headers-basic.any.js b/pkg/download/engine/testdata/wpt/fetch/api/headers/headers-basic.any.js new file mode 100644 index 00000000..ead10476 --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/headers/headers-basic.any.js @@ -0,0 +1,275 @@ +// META: title=Headers structure +// META: global=window,worker + +"use strict"; + +test(function() { + new Headers(); +}, "Create headers from no parameter"); + +test(function() { + new Headers(undefined); +}, "Create headers from undefined parameter"); + +test(function() { + new Headers({}); +}, "Create headers from empty object"); + +var parameters = [null, 1]; +parameters.forEach(function(parameter) { + test(function() { + assert_throws_js(TypeError, function() { new Headers(parameter) }); + }, "Create headers with " + parameter + " should throw"); +}); + +var headerDict = {"name1": "value1", + "name2": "value2", + "name3": "value3", + "name4": null, + "name5": undefined, + "name6": 1, + "Content-Type": "value4" +}; + +var headerSeq = []; +for (var name in headerDict) + headerSeq.push([name, headerDict[name]]); + +test(function() { + var headers = new Headers(headerSeq); + for (name in headerDict) { + assert_equals(headers.get(name), String(headerDict[name]), + "name: " + name + " has value: " + headerDict[name]); + } + assert_equals(headers.get("length"), null, "init should be treated as a sequence, not as a dictionary"); +}, "Create headers with sequence"); + +test(function() { + var headers = new Headers(headerDict); + for (name in headerDict) { + assert_equals(headers.get(name), String(headerDict[name]), + "name: " + name + " has value: " + headerDict[name]); + } +}, "Create headers with record"); + +test(function() { + var headers = new Headers(headerDict); + var headers2 = new Headers(headers); + for (name in headerDict) { + assert_equals(headers2.get(name), String(headerDict[name]), + "name: " + name + " has value: " + headerDict[name]); + } +}, "Create headers with existing headers"); + +test(function() { + var headers = new Headers() + headers[Symbol.iterator] = function *() { + yield ["test", "test"] + } + var headers2 = new Headers(headers) + assert_equals(headers2.get("test"), "test") +}, "Create headers with existing headers with custom iterator"); + +test(function() { + var headers = new Headers(); + for (name in headerDict) { + headers.append(name, headerDict[name]); + assert_equals(headers.get(name), String(headerDict[name]), + "name: " + name + " has value: " + headerDict[name]); + } +}, "Check append method"); + +test(function() { + var headers = new Headers(); + for (name in headerDict) { + headers.set(name, headerDict[name]); + assert_equals(headers.get(name), String(headerDict[name]), + "name: " + name + " has value: " + headerDict[name]); + } +}, "Check set method"); + +test(function() { + var headers = new Headers(headerDict); + for (name in headerDict) + assert_true(headers.has(name),"headers has name " + name); + + assert_false(headers.has("nameNotInHeaders"),"headers do not have header: nameNotInHeaders"); +}, "Check has method"); + +test(function() { + var headers = new Headers(headerDict); + for (name in headerDict) { + assert_true(headers.has(name),"headers have a header: " + name); + headers.delete(name) + assert_true(!headers.has(name),"headers do not have anymore a header: " + name); + } +}, "Check delete method"); + +test(function() { + var headers = new Headers(headerDict); + for (name in headerDict) + assert_equals(headers.get(name), String(headerDict[name]), + "name: " + name + " has value: " + headerDict[name]); + + assert_equals(headers.get("nameNotInHeaders"), null, "header: nameNotInHeaders has no value"); +}, "Check get method"); + +var headerEntriesDict = {"name1": "value1", + "Name2": "value2", + "name": "value3", + "content-Type": "value4", + "Content-Typ": "value5", + "Content-Types": "value6" +}; +var sortedHeaderDict = {}; +var headerValues = []; +var sortedHeaderKeys = Object.keys(headerEntriesDict).map(function(value) { + sortedHeaderDict[value.toLowerCase()] = headerEntriesDict[value]; + headerValues.push(headerEntriesDict[value]); + return value.toLowerCase(); +}).sort(); + +var iteratorPrototype = Object.getPrototypeOf(Object.getPrototypeOf([][Symbol.iterator]())); +function checkIteratorProperties(iterator) { + var prototype = Object.getPrototypeOf(iterator); + assert_equals(Object.getPrototypeOf(prototype), iteratorPrototype); + + var descriptor = Object.getOwnPropertyDescriptor(prototype, "next"); + assert_true(descriptor.configurable, "configurable"); + assert_true(descriptor.enumerable, "enumerable"); + assert_true(descriptor.writable, "writable"); +} + +test(function() { + var headers = new Headers(headerEntriesDict); + var actual = headers.keys(); + checkIteratorProperties(actual); + + sortedHeaderKeys.forEach(function(key) { + const entry = actual.next(); + assert_false(entry.done); + assert_equals(entry.value, key); + }); + assert_true(actual.next().done); + assert_true(actual.next().done); + + for (const key of headers.keys()) + assert_true(sortedHeaderKeys.indexOf(key) != -1); +}, "Check keys method"); + +test(function() { + var headers = new Headers(headerEntriesDict); + var actual = headers.values(); + checkIteratorProperties(actual); + + sortedHeaderKeys.forEach(function(key) { + const entry = actual.next(); + assert_false(entry.done); + assert_equals(entry.value, sortedHeaderDict[key]); + }); + assert_true(actual.next().done); + assert_true(actual.next().done); + + for (const value of headers.values()) + assert_true(headerValues.indexOf(value) != -1); +}, "Check values method"); + +test(function() { + var headers = new Headers(headerEntriesDict); + var actual = headers.entries(); + checkIteratorProperties(actual); + + sortedHeaderKeys.forEach(function(key) { + const entry = actual.next(); + assert_false(entry.done); + assert_equals(entry.value[0], key); + assert_equals(entry.value[1], sortedHeaderDict[key]); + }); + assert_true(actual.next().done); + assert_true(actual.next().done); + + for (const entry of headers.entries()) + assert_equals(entry[1], sortedHeaderDict[entry[0]]); +}, "Check entries method"); + +test(function() { + var headers = new Headers(headerEntriesDict); + var actual = headers[Symbol.iterator](); + + sortedHeaderKeys.forEach(function(key) { + const entry = actual.next(); + assert_false(entry.done); + assert_equals(entry.value[0], key); + assert_equals(entry.value[1], sortedHeaderDict[key]); + }); + assert_true(actual.next().done); + assert_true(actual.next().done); +}, "Check Symbol.iterator method"); + +test(function() { + var headers = new Headers(headerEntriesDict); + var reference = sortedHeaderKeys[Symbol.iterator](); + headers.forEach(function(value, key, container) { + assert_equals(headers, container); + const entry = reference.next(); + assert_false(entry.done); + assert_equals(key, entry.value); + assert_equals(value, sortedHeaderDict[entry.value]); + }); + assert_true(reference.next().done); +}, "Check forEach method"); + +test(() => { + const headers = new Headers({"foo": "2", "baz": "1", "BAR": "0"}); + const actualKeys = []; + const actualValues = []; + for (const [header, value] of headers) { + actualKeys.push(header); + actualValues.push(value); + headers.delete("foo"); + } + assert_array_equals(actualKeys, ["bar", "baz"]); + assert_array_equals(actualValues, ["0", "1"]); +}, "Iteration skips elements removed while iterating"); + +test(() => { + const headers = new Headers({"foo": "2", "baz": "1", "BAR": "0", "quux": "3"}); + const actualKeys = []; + const actualValues = []; + for (const [header, value] of headers) { + actualKeys.push(header); + actualValues.push(value); + if (header === "baz") + headers.delete("bar"); + } + assert_array_equals(actualKeys, ["bar", "baz", "quux"]); + assert_array_equals(actualValues, ["0", "1", "3"]); +}, "Removing elements already iterated over causes an element to be skipped during iteration"); + +test(() => { + const headers = new Headers({"foo": "2", "baz": "1", "BAR": "0", "quux": "3"}); + const actualKeys = []; + const actualValues = []; + for (const [header, value] of headers) { + actualKeys.push(header); + actualValues.push(value); + if (header === "baz") + headers.append("X-yZ", "4"); + } + assert_array_equals(actualKeys, ["bar", "baz", "foo", "quux", "x-yz"]); + assert_array_equals(actualValues, ["0", "1", "2", "3", "4"]); +}, "Appending a value pair during iteration causes it to be reached during iteration"); + +test(() => { + const headers = new Headers({"foo": "2", "baz": "1", "BAR": "0", "quux": "3"}); + const actualKeys = []; + const actualValues = []; + for (const [header, value] of headers) { + actualKeys.push(header); + actualValues.push(value); + if (header === "baz") + headers.append("abc", "-1"); + } + assert_array_equals(actualKeys, ["bar", "baz", "baz", "foo", "quux"]); + assert_array_equals(actualValues, ["0", "1", "1", "2", "3"]); +}, "Prepending a value pair before the current element position causes it to be skipped during iteration and adds the current element a second time"); diff --git a/pkg/download/engine/testdata/wpt/fetch/api/headers/headers-casing.any.js b/pkg/download/engine/testdata/wpt/fetch/api/headers/headers-casing.any.js new file mode 100644 index 00000000..20b8a9d3 --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/headers/headers-casing.any.js @@ -0,0 +1,54 @@ +// META: title=Headers case management +// META: global=window,worker + +"use strict"; + +var headerDictCase = {"UPPERCASE": "value1", + "lowercase": "value2", + "mixedCase": "value3", + "Content-TYPE": "value4" + }; + +function checkHeadersCase(originalName, headersToCheck, expectedDict) { + var lowCaseName = originalName.toLowerCase(); + var upCaseName = originalName.toUpperCase(); + var expectedValue = expectedDict[originalName]; + assert_equals(headersToCheck.get(originalName), expectedValue, + "name: " + originalName + " has value: " + expectedValue); + assert_equals(headersToCheck.get(lowCaseName), expectedValue, + "name: " + lowCaseName + " has value: " + expectedValue); + assert_equals(headersToCheck.get(upCaseName), expectedValue, + "name: " + upCaseName + " has value: " + expectedValue); +} + +test(function() { + var headers = new Headers(headerDictCase); + for (const name in headerDictCase) + checkHeadersCase(name, headers, headerDictCase) +}, "Create headers, names use characters with different case"); + +test(function() { + var headers = new Headers(); + for (const name in headerDictCase) { + headers.append(name, headerDictCase[name]); + checkHeadersCase(name, headers, headerDictCase); + } +}, "Check append method, names use characters with different case"); + +test(function() { + var headers = new Headers(); + for (const name in headerDictCase) { + headers.set(name, headerDictCase[name]); + checkHeadersCase(name, headers, headerDictCase); + } +}, "Check set method, names use characters with different case"); + +test(function() { + var headers = new Headers(); + for (const name in headerDictCase) + headers.set(name, headerDictCase[name]); + for (const name in headerDictCase) + headers.delete(name.toLowerCase()); + for (const name in headerDictCase) + assert_false(headers.has(name), "header " + name + " should have been deleted"); +}, "Check delete method, names use characters with different case"); diff --git a/pkg/download/engine/testdata/wpt/fetch/api/headers/headers-combine.any.js b/pkg/download/engine/testdata/wpt/fetch/api/headers/headers-combine.any.js new file mode 100644 index 00000000..4f3b6d11 --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/headers/headers-combine.any.js @@ -0,0 +1,66 @@ +// META: title=Headers have combined (and sorted) values +// META: global=window,worker + +"use strict"; + +var headerSeqCombine = [["single", "singleValue"], + ["double", "doubleValue1"], + ["double", "doubleValue2"], + ["triple", "tripleValue1"], + ["triple", "tripleValue2"], + ["triple", "tripleValue3"] +]; +var expectedDict = {"single": "singleValue", + "double": "doubleValue1, doubleValue2", + "triple": "tripleValue1, tripleValue2, tripleValue3" +}; + +test(function() { + var headers = new Headers(headerSeqCombine); + for (const name in expectedDict) + assert_equals(headers.get(name), expectedDict[name]); +}, "Create headers using same name for different values"); + +test(function() { + var headers = new Headers(headerSeqCombine); + for (const name in expectedDict) { + assert_true(headers.has(name), "name: " + name + " has value(s)"); + headers.delete(name); + assert_false(headers.has(name), "name: " + name + " has no value(s) anymore"); + } +}, "Check delete and has methods when using same name for different values"); + +test(function() { + var headers = new Headers(headerSeqCombine); + for (const name in expectedDict) { + headers.set(name,"newSingleValue"); + assert_equals(headers.get(name), "newSingleValue", "name: " + name + " has value: newSingleValue"); + } +}, "Check set methods when called with already used name"); + +test(function() { + var headers = new Headers(headerSeqCombine); + for (const name in expectedDict) { + var value = headers.get(name); + headers.append(name,"newSingleValue"); + assert_equals(headers.get(name), (value + ", " + "newSingleValue")); + } +}, "Check append methods when called with already used name"); + +test(() => { + const headers = new Headers([["1", "a"],["1", "b"]]); + for(let header of headers) { + assert_array_equals(header, ["1", "a, b"]); + } +}, "Iterate combined values"); + +test(() => { + const headers = new Headers([["2", "a"], ["1", "b"], ["2", "b"]]), + expected = [["1", "b"], ["2", "a, b"]]; + let i = 0; + for(let header of headers) { + assert_array_equals(header, expected[i]); + i++; + } + assert_equals(i, 2); +}, "Iterate combined values in sorted order") diff --git a/pkg/download/engine/testdata/wpt/fetch/api/headers/headers-errors.any.js b/pkg/download/engine/testdata/wpt/fetch/api/headers/headers-errors.any.js new file mode 100644 index 00000000..82dadd82 --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/headers/headers-errors.any.js @@ -0,0 +1,96 @@ +// META: title=Headers errors +// META: global=window,worker + +"use strict"; + +test(function() { + assert_throws_js(TypeError, function() { new Headers([["name"]]); }); +}, "Create headers giving an array having one string as init argument"); + +test(function() { + assert_throws_js(TypeError, function() { new Headers([["invalid", "invalidValue1", "invalidValue2"]]); }); +}, "Create headers giving an array having three strings as init argument"); + +test(function() { + assert_throws_js(TypeError, function() { new Headers([["invalidĀ", "Value1"]]); }); +}, "Create headers giving bad header name as init argument"); + +test(function() { + assert_throws_js(TypeError, function() { new Headers([["name", "invalidValueĀ"]]); }); +}, "Create headers giving bad header value as init argument"); + +var badNames = ["invalidĀ", {}]; +var badValues = ["invalidĀ"]; + +badNames.forEach(function(name) { + test(function() { + var headers = new Headers(); + assert_throws_js(TypeError, function() { headers.get(name); }); + }, "Check headers get with an invalid name " + name); +}); + +badNames.forEach(function(name) { + test(function() { + var headers = new Headers(); + assert_throws_js(TypeError, function() { headers.delete(name); }); + }, "Check headers delete with an invalid name " + name); +}); + +badNames.forEach(function(name) { + test(function() { + var headers = new Headers(); + assert_throws_js(TypeError, function() { headers.has(name); }); + }, "Check headers has with an invalid name " + name); +}); + +badNames.forEach(function(name) { + test(function() { + var headers = new Headers(); + assert_throws_js(TypeError, function() { headers.set(name, "Value1"); }); + }, "Check headers set with an invalid name " + name); +}); + +badValues.forEach(function(value) { + test(function() { + var headers = new Headers(); + assert_throws_js(TypeError, function() { headers.set("name", value); }); + }, "Check headers set with an invalid value " + value); +}); + +badNames.forEach(function(name) { + test(function() { + var headers = new Headers(); + assert_throws_js(TypeError, function() { headers.append("invalidĀ", "Value1"); }); + }, "Check headers append with an invalid name " + name); +}); + +badValues.forEach(function(value) { + test(function() { + var headers = new Headers(); + assert_throws_js(TypeError, function() { headers.append("name", value); }); + }, "Check headers append with an invalid value " + value); +}); + +test(function() { + var headers = new Headers([["name", "value"]]); + assert_throws_js(TypeError, function() { headers.forEach(); }); + assert_throws_js(TypeError, function() { headers.forEach(undefined); }); + assert_throws_js(TypeError, function() { headers.forEach(1); }); +}, "Headers forEach throws if argument is not callable"); + +test(function() { + var headers = new Headers([["name1", "value1"], ["name2", "value2"], ["name3", "value3"]]); + var counter = 0; + try { + headers.forEach(function(value, name) { + counter++; + if (name == "name2") + throw "error"; + }); + } catch (e) { + assert_equals(counter, 2); + assert_equals(e, "error"); + return; + } + assert_unreached(); +}, "Headers forEach loop should stop if callback is throwing exception"); diff --git a/pkg/download/engine/testdata/wpt/fetch/api/headers/headers-forbidden-override.any.js b/pkg/download/engine/testdata/wpt/fetch/api/headers/headers-forbidden-override.any.js new file mode 100644 index 00000000..7166e7a0 --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/headers/headers-forbidden-override.any.js @@ -0,0 +1,51 @@ +// META: global=window,worker + +let forbiddenMethods = [ + "TRACE", + "TRACK", + "CONNECT", + "trace", + "track", + "connect", + "\rtrace", + "\ttrack", + "\nconnect", + "trace,", + "GET,track ", + " connect", +]; + +let overrideHeaders = [ + "x-http-method-override", + "x-http-method", + "x-method-override", + "X-HTTP-METHOD-OVERRIDE", + "X-HTTP-METHOD", + "X-METHOD-OVERRIDE", +]; + +for (forbiddenMethod of forbiddenMethods) { + for (overrideHeader of overrideHeaders) { + test(() => { + let r = new Request("https://site.example/"); + r.headers.append(overrideHeader, forbiddenMethod); + assert_false(r.headers.has(overrideHeader)); + }, `header ${overrideHeader} is forbidden to use value ${forbiddenMethod}`); + } +} + +let permittedValues = [ + "GETTRACE", + "GET", + "\",TRACE\",", +]; + +for (permittedValue of permittedValues) { + for (overrideHeader of overrideHeaders) { + test(() => { + let r = new Request("https://site.example/"); + r.headers.append(overrideHeader, permittedValue); + assert_equals(permittedValue, r.headers.get(overrideHeader)); + }, `header ${overrideHeader} is allowed to use value ${permittedValue}`); + } +} diff --git a/pkg/download/engine/testdata/wpt/fetch/api/headers/headers-no-cors.any.js b/pkg/download/engine/testdata/wpt/fetch/api/headers/headers-no-cors.any.js new file mode 100644 index 00000000..60dbb9ef --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/headers/headers-no-cors.any.js @@ -0,0 +1,59 @@ +// META: global=window,worker + +"use strict"; + +promise_test(() => fetch("../cors/resources/not-cors-safelisted.json").then(res => res.json().then(runTests)), "Loading data…"); + +const longValue = "s".repeat(127); + +[ + { + "headers": ["accept", "accept-language", "content-language"], + "values": [longValue, "", longValue] + }, + { + "headers": ["accept", "accept-language", "content-language"], + "values": ["", longValue] + }, + { + "headers": ["content-type"], + "values": ["text/plain;" + "s".repeat(116), "text/plain"] + } +].forEach(testItem => { + testItem.headers.forEach(header => { + test(() => { + const noCorsHeaders = new Request("about:blank", { mode: "no-cors" }).headers; + testItem.values.forEach((value) => { + noCorsHeaders.append(header, value); + assert_equals(noCorsHeaders.get(header), testItem.values[0], '1'); + }); + noCorsHeaders.set(header, testItem.values.join(", ")); + assert_equals(noCorsHeaders.get(header), testItem.values[0], '2'); + noCorsHeaders.delete(header); + assert_false(noCorsHeaders.has(header)); + }, "\"no-cors\" Headers object cannot have " + header + " set to " + testItem.values.join(", ")); + }); +}); + +function runTests(testArray) { + testArray = testArray.concat([ + ["dpr", "2"], + ["rtt", "1.0"], + ["downlink", "-1.0"], + ["ect", "6g"], + ["save-data", "on"], + ["viewport-width", "100"], + ["width", "100"], + ["unknown", "doesitmatter"] + ]); + testArray.forEach(testItem => { + const [headerName, headerValue] = testItem; + test(() => { + const noCorsHeaders = new Request("about:blank", { mode: "no-cors" }).headers; + noCorsHeaders.append(headerName, headerValue); + assert_false(noCorsHeaders.has(headerName)); + noCorsHeaders.set(headerName, headerValue); + assert_false(noCorsHeaders.has(headerName)); + }, "\"no-cors\" Headers object cannot have " + headerName + "/" + headerValue + " as header"); + }); +} diff --git a/pkg/download/engine/testdata/wpt/fetch/api/headers/headers-normalize.any.js b/pkg/download/engine/testdata/wpt/fetch/api/headers/headers-normalize.any.js new file mode 100644 index 00000000..68cf5b85 --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/headers/headers-normalize.any.js @@ -0,0 +1,56 @@ +// META: title=Headers normalize values +// META: global=window,worker + +"use strict"; + +const expectations = { + "name1": [" space ", "space"], + "name2": ["\ttab\t", "tab"], + "name3": [" spaceAndTab\t", "spaceAndTab"], + "name4": ["\r\n newLine", "newLine"], //obs-fold cases + "name5": ["newLine\r\n ", "newLine"], + "name6": ["\r\n\tnewLine", "newLine"], + "name7": ["\t\f\tnewLine\n", "\f\tnewLine"], + "name8": ["newLine\xa0", "newLine\xa0"], // \xa0 == non breaking space +}; + +test(function () { + const headerDict = Object.fromEntries( + Object.entries(expectations).map(([name, [actual]]) => [name, actual]), + ); + var headers = new Headers(headerDict); + for (const name in expectations) { + const expected = expectations[name][1]; + assert_equals( + headers.get(name), + expected, + "name: " + name + " has normalized value: " + expected, + ); + } +}, "Create headers with not normalized values"); + +test(function () { + var headers = new Headers(); + for (const name in expectations) { + headers.append(name, expectations[name][0]); + const expected = expectations[name][1]; + assert_equals( + headers.get(name), + expected, + "name: " + name + " has value: " + expected, + ); + } +}, "Check append method with not normalized values"); + +test(function () { + var headers = new Headers(); + for (const name in expectations) { + headers.set(name, expectations[name][0]); + const expected = expectations[name][1]; + assert_equals( + headers.get(name), + expected, + "name: " + name + " has value: " + expected, + ); + } +}, "Check set method with not normalized values"); diff --git a/pkg/download/engine/testdata/wpt/fetch/api/headers/headers-record.any.js b/pkg/download/engine/testdata/wpt/fetch/api/headers/headers-record.any.js new file mode 100644 index 00000000..fa853914 --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/headers/headers-record.any.js @@ -0,0 +1,357 @@ +// META: global=window,worker + +"use strict"; + +var log = []; +function clearLog() { + log = []; +} +function addLogEntry(name, args) { + log.push([ name, ...args ]); +} + +var loggingHandler = { +}; + +setup(function() { + for (let prop of Object.getOwnPropertyNames(Reflect)) { + loggingHandler[prop] = function(...args) { + addLogEntry(prop, args); + return Reflect[prop](...args); + } + } +}); + +test(function() { + var h = new Headers(); + assert_equals([...h].length, 0); +}, "Passing nothing to Headers constructor"); + +test(function() { + var h = new Headers(undefined); + assert_equals([...h].length, 0); +}, "Passing undefined to Headers constructor"); + +test(function() { + assert_throws_js(TypeError, function() { + var h = new Headers(null); + }); +}, "Passing null to Headers constructor"); + +test(function() { + this.add_cleanup(clearLog); + var record = { a: "b" }; + var proxy = new Proxy(record, loggingHandler); + var h = new Headers(proxy); + + assert_equals(log.length, 4); + // The first thing is the [[Get]] of Symbol.iterator to figure out whether + // we're a sequence, during overload resolution. + assert_array_equals(log[0], ["get", record, Symbol.iterator, proxy]); + // Then we have the [[OwnPropertyKeys]] from + // https://webidl.spec.whatwg.org/#es-to-record step 4. + assert_array_equals(log[1], ["ownKeys", record]); + // Then the [[GetOwnProperty]] from step 5.1. + assert_array_equals(log[2], ["getOwnPropertyDescriptor", record, "a"]); + // Then the [[Get]] from step 5.2. + assert_array_equals(log[3], ["get", record, "a", proxy]); + + // Check the results. + assert_equals([...h].length, 1); + assert_array_equals([...h.keys()], ["a"]); + assert_true(h.has("a")); + assert_equals(h.get("a"), "b"); +}, "Basic operation with one property"); + +test(function() { + this.add_cleanup(clearLog); + var recordProto = { c: "d" }; + var record = Object.create(recordProto, { a: { value: "b", enumerable: true } }); + var proxy = new Proxy(record, loggingHandler); + var h = new Headers(proxy); + + assert_equals(log.length, 4); + // The first thing is the [[Get]] of Symbol.iterator to figure out whether + // we're a sequence, during overload resolution. + assert_array_equals(log[0], ["get", record, Symbol.iterator, proxy]); + // Then we have the [[OwnPropertyKeys]] from + // https://webidl.spec.whatwg.org/#es-to-record step 4. + assert_array_equals(log[1], ["ownKeys", record]); + // Then the [[GetOwnProperty]] from step 5.1. + assert_array_equals(log[2], ["getOwnPropertyDescriptor", record, "a"]); + // Then the [[Get]] from step 5.2. + assert_array_equals(log[3], ["get", record, "a", proxy]); + + // Check the results. + assert_equals([...h].length, 1); + assert_array_equals([...h.keys()], ["a"]); + assert_true(h.has("a")); + assert_equals(h.get("a"), "b"); +}, "Basic operation with one property and a proto"); + +test(function() { + this.add_cleanup(clearLog); + var record = { a: "b", c: "d" }; + var proxy = new Proxy(record, loggingHandler); + var h = new Headers(proxy); + + assert_equals(log.length, 6); + // The first thing is the [[Get]] of Symbol.iterator to figure out whether + // we're a sequence, during overload resolution. + assert_array_equals(log[0], ["get", record, Symbol.iterator, proxy]); + // Then we have the [[OwnPropertyKeys]] from + // https://webidl.spec.whatwg.org/#es-to-record step 4. + assert_array_equals(log[1], ["ownKeys", record]); + // Then the [[GetOwnProperty]] from step 5.1. + assert_array_equals(log[2], ["getOwnPropertyDescriptor", record, "a"]); + // Then the [[Get]] from step 5.2. + assert_array_equals(log[3], ["get", record, "a", proxy]); + // Then the second [[GetOwnProperty]] from step 5.1. + assert_array_equals(log[4], ["getOwnPropertyDescriptor", record, "c"]); + // Then the second [[Get]] from step 5.2. + assert_array_equals(log[5], ["get", record, "c", proxy]); + + // Check the results. + assert_equals([...h].length, 2); + assert_array_equals([...h.keys()], ["a", "c"]); + assert_true(h.has("a")); + assert_equals(h.get("a"), "b"); + assert_true(h.has("c")); + assert_equals(h.get("c"), "d"); +}, "Correct operation ordering with two properties"); + +test(function() { + this.add_cleanup(clearLog); + var record = { a: "b", "\uFFFF": "d" }; + var proxy = new Proxy(record, loggingHandler); + assert_throws_js(TypeError, function() { + var h = new Headers(proxy); + }); + + assert_equals(log.length, 5); + // The first thing is the [[Get]] of Symbol.iterator to figure out whether + // we're a sequence, during overload resolution. + assert_array_equals(log[0], ["get", record, Symbol.iterator, proxy]); + // Then we have the [[OwnPropertyKeys]] from + // https://webidl.spec.whatwg.org/#es-to-record step 4. + assert_array_equals(log[1], ["ownKeys", record]); + // Then the [[GetOwnProperty]] from step 5.1. + assert_array_equals(log[2], ["getOwnPropertyDescriptor", record, "a"]); + // Then the [[Get]] from step 5.2. + assert_array_equals(log[3], ["get", record, "a", proxy]); + // Then the second [[GetOwnProperty]] from step 5.1. + assert_array_equals(log[4], ["getOwnPropertyDescriptor", record, "\uFFFF"]); + // The second [[Get]] never happens, because we convert the invalid name to a + // ByteString first and throw. +}, "Correct operation ordering with two properties one of which has an invalid name"); + +test(function() { + this.add_cleanup(clearLog); + var record = { a: "\uFFFF", c: "d" } + var proxy = new Proxy(record, loggingHandler); + assert_throws_js(TypeError, function() { + var h = new Headers(proxy); + }); + + assert_equals(log.length, 4); + // The first thing is the [[Get]] of Symbol.iterator to figure out whether + // we're a sequence, during overload resolution. + assert_array_equals(log[0], ["get", record, Symbol.iterator, proxy]); + // Then we have the [[OwnPropertyKeys]] from + // https://webidl.spec.whatwg.org/#es-to-record step 4. + assert_array_equals(log[1], ["ownKeys", record]); + // Then the [[GetOwnProperty]] from step 5.1. + assert_array_equals(log[2], ["getOwnPropertyDescriptor", record, "a"]); + // Then the [[Get]] from step 5.2. + assert_array_equals(log[3], ["get", record, "a", proxy]); + // Nothing else after this, because converting the result of that [[Get]] to a + // ByteString throws. +}, "Correct operation ordering with two properties one of which has an invalid value"); + +test(function() { + this.add_cleanup(clearLog); + var record = {}; + Object.defineProperty(record, "a", { value: "b", enumerable: false }); + Object.defineProperty(record, "c", { value: "d", enumerable: true }); + Object.defineProperty(record, "e", { value: "f", enumerable: false }); + var proxy = new Proxy(record, loggingHandler); + var h = new Headers(proxy); + + assert_equals(log.length, 6); + // The first thing is the [[Get]] of Symbol.iterator to figure out whether + // we're a sequence, during overload resolution. + assert_array_equals(log[0], ["get", record, Symbol.iterator, proxy]); + // Then we have the [[OwnPropertyKeys]] from + // https://webidl.spec.whatwg.org/#es-to-record step 4. + assert_array_equals(log[1], ["ownKeys", record]); + // Then the [[GetOwnProperty]] from step 5.1. + assert_array_equals(log[2], ["getOwnPropertyDescriptor", record, "a"]); + // No [[Get]] because not enumerable + // Then the second [[GetOwnProperty]] from step 5.1. + assert_array_equals(log[3], ["getOwnPropertyDescriptor", record, "c"]); + // Then the [[Get]] from step 5.2. + assert_array_equals(log[4], ["get", record, "c", proxy]); + // Then the third [[GetOwnProperty]] from step 5.1. + assert_array_equals(log[5], ["getOwnPropertyDescriptor", record, "e"]); + // No [[Get]] because not enumerable + + // Check the results. + assert_equals([...h].length, 1); + assert_array_equals([...h.keys()], ["c"]); + assert_true(h.has("c")); + assert_equals(h.get("c"), "d"); +}, "Correct operation ordering with non-enumerable properties"); + +test(function() { + this.add_cleanup(clearLog); + var record = {a: "b", c: "d", e: "f"}; + var lyingHandler = { + getOwnPropertyDescriptor: function(target, name) { + if (name == "a" || name == "e") { + return undefined; + } + return Reflect.getOwnPropertyDescriptor(target, name); + } + }; + var lyingProxy = new Proxy(record, lyingHandler); + var proxy = new Proxy(lyingProxy, loggingHandler); + var h = new Headers(proxy); + + assert_equals(log.length, 6); + // The first thing is the [[Get]] of Symbol.iterator to figure out whether + // we're a sequence, during overload resolution. + assert_array_equals(log[0], ["get", lyingProxy, Symbol.iterator, proxy]); + // Then we have the [[OwnPropertyKeys]] from + // https://webidl.spec.whatwg.org/#es-to-record step 4. + assert_array_equals(log[1], ["ownKeys", lyingProxy]); + // Then the [[GetOwnProperty]] from step 5.1. + assert_array_equals(log[2], ["getOwnPropertyDescriptor", lyingProxy, "a"]); + // No [[Get]] because no descriptor + // Then the second [[GetOwnProperty]] from step 5.1. + assert_array_equals(log[3], ["getOwnPropertyDescriptor", lyingProxy, "c"]); + // Then the [[Get]] from step 5.2. + assert_array_equals(log[4], ["get", lyingProxy, "c", proxy]); + // Then the third [[GetOwnProperty]] from step 5.1. + assert_array_equals(log[5], ["getOwnPropertyDescriptor", lyingProxy, "e"]); + // No [[Get]] because no descriptor + + // Check the results. + assert_equals([...h].length, 1); + assert_array_equals([...h.keys()], ["c"]); + assert_true(h.has("c")); + assert_equals(h.get("c"), "d"); +}, "Correct operation ordering with undefined descriptors"); + +test(function() { + this.add_cleanup(clearLog); + var record = {a: "b", c: "d"}; + var lyingHandler = { + ownKeys: function() { + return [ "a", "c", "a", "c" ]; + }, + }; + var lyingProxy = new Proxy(record, lyingHandler); + var proxy = new Proxy(lyingProxy, loggingHandler); + + // Returning duplicate keys from ownKeys() throws a TypeError. + assert_throws_js(TypeError, + function() { var h = new Headers(proxy); }); + + assert_equals(log.length, 2); + // The first thing is the [[Get]] of Symbol.iterator to figure out whether + // we're a sequence, during overload resolution. + assert_array_equals(log[0], ["get", lyingProxy, Symbol.iterator, proxy]); + // Then we have the [[OwnPropertyKeys]] from + // https://webidl.spec.whatwg.org/#es-to-record step 4. + assert_array_equals(log[1], ["ownKeys", lyingProxy]); +}, "Correct operation ordering with repeated keys"); + +test(function() { + this.add_cleanup(clearLog); + var record = { + a: "b", + [Symbol.toStringTag]: { + // Make sure the ToString conversion of the value happens + // after the ToString conversion of the key. + toString: function () { addLogEntry("toString", [this]); return "nope"; } + }, + c: "d" }; + var proxy = new Proxy(record, loggingHandler); + assert_throws_js(TypeError, + function() { var h = new Headers(proxy); }); + + assert_equals(log.length, 7); + // The first thing is the [[Get]] of Symbol.iterator to figure out whether + // we're a sequence, during overload resolution. + assert_array_equals(log[0], ["get", record, Symbol.iterator, proxy]); + // Then we have the [[OwnPropertyKeys]] from + // https://webidl.spec.whatwg.org/#es-to-record step 4. + assert_array_equals(log[1], ["ownKeys", record]); + // Then the [[GetOwnProperty]] from step 5.1. + assert_array_equals(log[2], ["getOwnPropertyDescriptor", record, "a"]); + // Then the [[Get]] from step 5.2. + assert_array_equals(log[3], ["get", record, "a", proxy]); + // Then the second [[GetOwnProperty]] from step 5.1. + assert_array_equals(log[4], ["getOwnPropertyDescriptor", record, "c"]); + // Then the second [[Get]] from step 5.2. + assert_array_equals(log[5], ["get", record, "c", proxy]); + // Then the third [[GetOwnProperty]] from step 5.1. + assert_array_equals(log[6], ["getOwnPropertyDescriptor", record, + Symbol.toStringTag]); + // Then we throw an exception converting the Symbol to a string, before we do + // the third [[Get]]. +}, "Basic operation with Symbol keys"); + +test(function() { + this.add_cleanup(clearLog); + var record = { + a: { + toString: function() { addLogEntry("toString", [this]); return "b"; } + }, + [Symbol.toStringTag]: { + toString: function () { addLogEntry("toString", [this]); return "nope"; } + }, + c: { + toString: function() { addLogEntry("toString", [this]); return "d"; } + } + }; + // Now make that Symbol-named property not enumerable. + Object.defineProperty(record, Symbol.toStringTag, { enumerable: false }); + assert_array_equals(Reflect.ownKeys(record), + ["a", "c", Symbol.toStringTag]); + + var proxy = new Proxy(record, loggingHandler); + var h = new Headers(proxy); + + assert_equals(log.length, 9); + // The first thing is the [[Get]] of Symbol.iterator to figure out whether + // we're a sequence, during overload resolution. + assert_array_equals(log[0], ["get", record, Symbol.iterator, proxy]); + // Then we have the [[OwnPropertyKeys]] from + // https://webidl.spec.whatwg.org/#es-to-record step 4. + assert_array_equals(log[1], ["ownKeys", record]); + // Then the [[GetOwnProperty]] from step 5.1. + assert_array_equals(log[2], ["getOwnPropertyDescriptor", record, "a"]); + // Then the [[Get]] from step 5.2. + assert_array_equals(log[3], ["get", record, "a", proxy]); + // Then the ToString on the value. + assert_array_equals(log[4], ["toString", record.a]); + // Then the second [[GetOwnProperty]] from step 5.1. + assert_array_equals(log[5], ["getOwnPropertyDescriptor", record, "c"]); + // Then the second [[Get]] from step 5.2. + assert_array_equals(log[6], ["get", record, "c", proxy]); + // Then the ToString on the value. + assert_array_equals(log[7], ["toString", record.c]); + // Then the third [[GetOwnProperty]] from step 5.1. + assert_array_equals(log[8], ["getOwnPropertyDescriptor", record, + Symbol.toStringTag]); + // No [[Get]] because not enumerable. + + // Check the results. + assert_equals([...h].length, 2); + assert_array_equals([...h.keys()], ["a", "c"]); + assert_true(h.has("a")); + assert_equals(h.get("a"), "b"); + assert_true(h.has("c")); + assert_equals(h.get("c"), "d"); +}, "Operation with non-enumerable Symbol keys"); diff --git a/pkg/download/engine/testdata/wpt/fetch/api/headers/headers-structure.any.js b/pkg/download/engine/testdata/wpt/fetch/api/headers/headers-structure.any.js new file mode 100644 index 00000000..d826bcab --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/headers/headers-structure.any.js @@ -0,0 +1,20 @@ +// META: title=Headers basic +// META: global=window,worker + +"use strict"; + +var headers = new Headers(); +var methods = ["append", + "delete", + "get", + "has", + "set", + //Headers is iterable + "entries", + "keys", + "values" + ]; +for (var idx in methods) + test(function() { + assert_true(methods[idx] in headers, "headers has " + methods[idx] + " method"); + }, "Headers has " + methods[idx] + " method"); diff --git a/pkg/download/engine/testdata/wpt/fetch/api/request/forbidden-method.any.js b/pkg/download/engine/testdata/wpt/fetch/api/request/forbidden-method.any.js new file mode 100644 index 00000000..eb13f37f --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/request/forbidden-method.any.js @@ -0,0 +1,13 @@ +// META: global=window,worker + +// https://fetch.spec.whatwg.org/#forbidden-method +for (const method of [ + 'CONNECT', 'TRACE', 'TRACK', + 'connect', 'trace', 'track' + ]) { + test(function() { + assert_throws_js(TypeError, + function() { new Request('./', {method: method}); } + ); + }, 'Request() with a forbidden method ' + method + ' must throw.'); +} diff --git a/pkg/download/engine/testdata/wpt/fetch/api/request/request-clone-readable-stream-body.any.js b/pkg/download/engine/testdata/wpt/fetch/api/request/request-clone-readable-stream-body.any.js new file mode 100644 index 00000000..167715ae --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/request/request-clone-readable-stream-body.any.js @@ -0,0 +1,22 @@ +// META: global=window,worker + +promise_test(async () => { + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("hello")); + controller.close(); + } + }); + + const r1 = new Request("https://example.com/", { method: "POST", body: stream, duplex: "half" }); + const r2 = r1.clone(); + assert_false(r2.bodyUsed, "clone should not be marked as used"); + assert_not_equals(r2.body, null, "clone should have a body"); + + // Constructing a new Request from the clone should preserve the body. + const r3 = new Request(r2); + assert_not_equals(r3.body, null, "Request constructed from clone should have a body"); + + const text = await r3.text(); + assert_equals(text, "hello", "body content should be preserved through clone() and new Request()"); +}, "new Request(clone) preserves a ReadableStream body that came from clone()"); diff --git a/pkg/download/engine/testdata/wpt/fetch/api/request/request-constructor-init-body-override.any.js b/pkg/download/engine/testdata/wpt/fetch/api/request/request-constructor-init-body-override.any.js new file mode 100644 index 00000000..c2bbf86f --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/request/request-constructor-init-body-override.any.js @@ -0,0 +1,37 @@ +promise_test(async function () { + const req1 = new Request("https://example.com/", { + body: "req1", + method: "POST", + }); + + const text1 = await req1.text(); + assert_equals( + text1, + "req1", + "The body of the first request should be 'req1'." + ); + + const req2 = new Request(req1, { body: "req2" }); + const bodyText = await req2.text(); + assert_equals( + bodyText, + "req2", + "The body of the second request should be overridden to 'req2'." + ); + +}, "Check that the body of a new request can be overridden when created from an existing Request object"); + +promise_test(async function () { + const req1 = new Request("https://example.com/", { + body: "req1", + method: "POST", + }); + + const req2 = new Request("https://example.com/", req1); + const bodyText = await req2.text(); + assert_equals( + bodyText, + "req1", + "The body of the second request should be the same as the first." + ); +}, "Check that the body of a new request can be duplicated from an existing Request object"); diff --git a/pkg/download/engine/testdata/wpt/fetch/api/request/request-consume-empty.any.js b/pkg/download/engine/testdata/wpt/fetch/api/request/request-consume-empty.any.js new file mode 100644 index 00000000..0bf9672a --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/request/request-consume-empty.any.js @@ -0,0 +1,89 @@ +// META: global=window,worker +// META: title=Request consume empty bodies + +function checkBodyText(test, request) { + return request.text().then(function(bodyAsText) { + assert_equals(bodyAsText, "", "Resolved value should be empty"); + assert_false(request.bodyUsed); + }); +} + +async function checkBodyBlob(test, request) { + const bodyAsBlob = await request.blob(); + const body = await bodyAsBlob.text(); + assert_equals(body, "", "Resolved value should be empty"); + assert_false(request.bodyUsed); +} + +function checkBodyArrayBuffer(test, request) { + return request.arrayBuffer().then(function(bodyAsArrayBuffer) { + assert_equals(bodyAsArrayBuffer.byteLength, 0, "Resolved value should be empty"); + assert_false(request.bodyUsed); + }); +} + +function checkBodyJSON(test, request) { + return request.json().then( + function(bodyAsJSON) { + assert_unreached("JSON parsing should fail"); + }, + function() { + assert_false(request.bodyUsed); + }); +} + +function checkBodyFormData(test, request) { + return request.formData().then(function(bodyAsFormData) { + assert_true(bodyAsFormData instanceof FormData, "Should receive a FormData"); + assert_false(request.bodyUsed); + }); +} + +function checkBodyFormDataError(test, request) { + return promise_rejects_js(test, TypeError, request.formData()).then(function() { + assert_false(request.bodyUsed); + }); +} + +function checkRequestWithNoBody(bodyType, checkFunction, headers = []) { + promise_test(function(test) { + var request = new Request("", {"method": "POST", "headers": headers}); + assert_false(request.bodyUsed); + return checkFunction(test, request); + }, "Consume request's body as " + bodyType); +} + +checkRequestWithNoBody("text", checkBodyText); +checkRequestWithNoBody("blob", checkBodyBlob); +checkRequestWithNoBody("arrayBuffer", checkBodyArrayBuffer); +checkRequestWithNoBody("json (error case)", checkBodyJSON); +checkRequestWithNoBody("formData with correct multipart type (error case)", checkBodyFormDataError, [["Content-Type", 'multipart/form-data; boundary="boundary"']]); +checkRequestWithNoBody("formData with correct urlencoded type", checkBodyFormData, [["Content-Type", "application/x-www-form-urlencoded;charset=UTF-8"]]); +checkRequestWithNoBody("formData without correct type (error case)", checkBodyFormDataError); + +function checkRequestWithEmptyBody(bodyType, body, asText) { + promise_test(function(test) { + var request = new Request("", {"method": "POST", "body": body}); + assert_false(request.bodyUsed, "bodyUsed is false at init"); + if (asText) { + return request.text().then(function(bodyAsString) { + assert_equals(bodyAsString.length, 0, "Resolved value should be empty"); + assert_true(request.bodyUsed, "bodyUsed is true after being consumed"); + }); + } + return request.arrayBuffer().then(function(bodyAsArrayBuffer) { + assert_equals(bodyAsArrayBuffer.byteLength, 0, "Resolved value should be empty"); + assert_true(request.bodyUsed, "bodyUsed is true after being consumed"); + }); + }, "Consume empty " + bodyType + " request body as " + (asText ? "text" : "arrayBuffer")); +} + +// FIXME: Add BufferSource, FormData and URLSearchParams. +checkRequestWithEmptyBody("blob", new Blob([], { "type" : "text/plain" }), false); +checkRequestWithEmptyBody("text", "", false); +checkRequestWithEmptyBody("blob", new Blob([], { "type" : "text/plain" }), true); +checkRequestWithEmptyBody("text", "", true); +checkRequestWithEmptyBody("URLSearchParams", new URLSearchParams(""), true); +// FIXME: This test assumes that the empty string be returned but it is not clear whether that is right. See https://github.com/web-platform-tests/wpt/pull/3950. +checkRequestWithEmptyBody("FormData", new FormData(), true); +checkRequestWithEmptyBody("ArrayBuffer", new ArrayBuffer(), true); diff --git a/pkg/download/engine/testdata/wpt/fetch/api/request/request-consume.any.js b/pkg/download/engine/testdata/wpt/fetch/api/request/request-consume.any.js new file mode 100644 index 00000000..b4cbe745 --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/request/request-consume.any.js @@ -0,0 +1,148 @@ +// META: global=window,worker +// META: title=Request consume +// META: script=../resources/utils.js + +function checkBodyText(request, expectedBody) { + return request.text().then(function(bodyAsText) { + assert_equals(bodyAsText, expectedBody, "Retrieve and verify request's body"); + assert_true(request.bodyUsed, "body as text: bodyUsed turned true"); + }); +} + +async function checkBodyBlob(request, expectedBody, checkContentType) { + const bodyAsBlob = await request.blob(); + + if (checkContentType) + assert_equals(bodyAsBlob.type, "text/plain", "Blob body type should be computed from the request Content-Type"); + + const body = await bodyAsBlob.text(); + assert_equals(body, expectedBody, "Retrieve and verify request's body"); + assert_true(request.bodyUsed, "body as blob: bodyUsed turned true"); +} + +function checkBodyArrayBuffer(request, expectedBody) { + return request.arrayBuffer().then(function(bodyAsArrayBuffer) { + validateBufferFromString(bodyAsArrayBuffer, expectedBody, "Retrieve and verify request's body"); + assert_true(request.bodyUsed, "body as arrayBuffer: bodyUsed turned true"); + }); +} + +function checkBodyBytes(request, expectedBody) { + return request.bytes().then(function(bodyAsUint8Array) { + assert_true(bodyAsUint8Array instanceof Uint8Array); + validateBufferFromString(bodyAsUint8Array.buffer, expectedBody, "Retrieve and verify request's body"); + assert_true(request.bodyUsed, "body as bytes: bodyUsed turned true"); + }); +} + +function checkBodyJSON(request, expectedBody) { + return request.json().then(function(bodyAsJSON) { + var strBody = JSON.stringify(bodyAsJSON) + assert_equals(strBody, expectedBody, "Retrieve and verify request's body"); + assert_true(request.bodyUsed, "body as json: bodyUsed turned true"); + }); +} + +function checkBodyFormData(request, expectedBody) { + return request.formData().then(function(bodyAsFormData) { + assert_true(bodyAsFormData instanceof FormData, "Should receive a FormData"); + assert_true(request.bodyUsed, "body as formData: bodyUsed turned true"); + }); +} + +function checkRequestBody(body, expected, bodyType) { + promise_test(function(test) { + var request = new Request("", {"method": "POST", "body": body, "headers": [["Content-Type", "text/PLAIN"]] }); + assert_false(request.bodyUsed, "bodyUsed is false at init"); + return checkBodyText(request, expected); + }, "Consume " + bodyType + " request's body as text"); + promise_test(function(test) { + var request = new Request("", {"method": "POST", "body": body }); + assert_false(request.bodyUsed, "bodyUsed is false at init"); + return checkBodyBlob(request, expected); + }, "Consume " + bodyType + " request's body as blob"); + promise_test(function(test) { + var request = new Request("", {"method": "POST", "body": body }); + assert_false(request.bodyUsed, "bodyUsed is false at init"); + return checkBodyArrayBuffer(request, expected); + }, "Consume " + bodyType + " request's body as arrayBuffer"); + promise_test(function(test) { + var request = new Request("", {"method": "POST", "body": body }); + assert_false(request.bodyUsed, "bodyUsed is false at init"); + return checkBodyBytes(request, expected); + }, "Consume " + bodyType + " request's body as bytes"); + promise_test(function(test) { + var request = new Request("", {"method": "POST", "body": body }); + assert_false(request.bodyUsed, "bodyUsed is false at init"); + return checkBodyJSON(request, expected); + }, "Consume " + bodyType + " request's body as JSON"); +} + +var textData = JSON.stringify("This is response's body"); +var blob = new Blob([textData], { "type" : "text/plain" }); + +checkRequestBody(textData, textData, "String"); + +var string = "\"123456\""; +function getArrayBuffer() { + var arrayBuffer = new ArrayBuffer(8); + var int8Array = new Int8Array(arrayBuffer); + for (var cptr = 0; cptr < 8; cptr++) + int8Array[cptr] = string.charCodeAt(cptr); + return arrayBuffer; +} + +function getArrayBufferWithZeros() { + var arrayBuffer = new ArrayBuffer(10); + var int8Array = new Int8Array(arrayBuffer); + for (var cptr = 0; cptr < 8; cptr++) + int8Array[cptr + 1] = string.charCodeAt(cptr); + return arrayBuffer; +} + +checkRequestBody(getArrayBuffer(), string, "ArrayBuffer"); +checkRequestBody(new Uint8Array(getArrayBuffer()), string, "Uint8Array"); +checkRequestBody(new Int8Array(getArrayBufferWithZeros(), 1, 8), string, "Int8Array"); +checkRequestBody(new Float32Array(getArrayBuffer()), string, "Float32Array"); +checkRequestBody(new DataView(getArrayBufferWithZeros(), 1, 8), string, "DataView"); + +promise_test(function(test) { + var formData = new FormData(); + formData.append("name", "value") + var request = new Request("", {"method": "POST", "body": formData }); + assert_false(request.bodyUsed, "bodyUsed is false at init"); + return checkBodyFormData(request, formData); +}, "Consume FormData request's body as FormData"); + +function checkBlobResponseBody(blobBody, blobData, bodyType, checkFunction) { + promise_test(function(test) { + var response = new Response(blobBody); + assert_false(response.bodyUsed, "bodyUsed is false at init"); + return checkFunction(response, blobData); + }, "Consume blob response's body as " + bodyType); +} + +checkBlobResponseBody(blob, textData, "blob", checkBodyBlob); +checkBlobResponseBody(blob, textData, "text", checkBodyText); +checkBlobResponseBody(blob, textData, "json", checkBodyJSON); +checkBlobResponseBody(blob, textData, "arrayBuffer", checkBodyArrayBuffer); +checkBlobResponseBody(blob, textData, "bytes", checkBodyBytes); +checkBlobResponseBody(new Blob([""]), "", "blob (empty blob as input)", checkBodyBlob); + +var goodJSONValues = ["null", "1", "true", "\"string\""]; +goodJSONValues.forEach(function(value) { + promise_test(function(test) { + var request = new Request("", {"method": "POST", "body": value}); + return request.json().then(function(v) { + assert_equals(v, JSON.parse(value)); + }); + }, "Consume JSON from text: '" + JSON.stringify(value) + "'"); +}); + +var badJSONValues = ["undefined", "{", "a", "["]; +badJSONValues.forEach(function(value) { + promise_test(function(test) { + var request = new Request("", {"method": "POST", "body": value}); + return promise_rejects_js(test, SyntaxError, request.json()); + }, "Trying to consume bad JSON text as JSON: '" + value + "'"); +}); diff --git a/pkg/download/engine/testdata/wpt/fetch/api/request/request-disturbed.any.js b/pkg/download/engine/testdata/wpt/fetch/api/request/request-disturbed.any.js new file mode 100644 index 00000000..8a11de78 --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/request/request-disturbed.any.js @@ -0,0 +1,109 @@ +// META: global=window,worker +// META: title=Request disturbed +// META: script=../resources/utils.js + +var initValuesDict = {"method" : "POST", + "body" : "Request's body" +}; + +var noBodyConsumed = new Request(""); +var bodyConsumed = new Request("", initValuesDict); + +test(() => { + assert_equals(noBodyConsumed.body, null, "body's default value is null"); + assert_false(noBodyConsumed.bodyUsed , "bodyUsed is false when request is not disturbed"); + assert_not_equals(bodyConsumed.body, null, "non-null body"); + assert_true(bodyConsumed.body instanceof ReadableStream, "non-null body type"); + assert_false(noBodyConsumed.bodyUsed, "bodyUsed is false when request is not disturbed"); +}, "Request's body: initial state"); + +noBodyConsumed.blob(); +bodyConsumed.blob(); + +test(function() { + assert_false(noBodyConsumed.bodyUsed , "bodyUsed is false when request is not disturbed"); + try { + noBodyConsumed.clone(); + } catch (e) { + assert_unreached("Can use request not disturbed for creating or cloning request"); + } +}, "Request without body cannot be disturbed"); + +test(function() { + assert_true(bodyConsumed.bodyUsed , "bodyUsed is true when request is disturbed"); + assert_throws_js(TypeError, function() { bodyConsumed.clone(); }); +}, "Check cloning a disturbed request"); + +test(function() { + assert_true(bodyConsumed.bodyUsed , "bodyUsed is true when request is disturbed"); + assert_throws_js(TypeError, function() { new Request(bodyConsumed); }); +}, "Check creating a new request from a disturbed request"); + +promise_test(function() { + assert_true(bodyConsumed.bodyUsed , "bodyUsed is true when request is disturbed"); + const originalBody = bodyConsumed.body; + const bodyReplaced = new Request(bodyConsumed, { body: "Replaced body" }); + assert_not_equals(bodyReplaced.body, originalBody, "new request's body is new"); + assert_false(bodyReplaced.bodyUsed, "bodyUsed is false when request is not disturbed"); + return bodyReplaced.text().then(text => { + assert_equals(text, "Replaced body"); + }); +}, "Check creating a new request with a new body from a disturbed request"); + +promise_test(function() { + var bodyRequest = new Request("", initValuesDict); + const originalBody = bodyRequest.body; + assert_false(bodyRequest.bodyUsed , "bodyUsed is false when request is not disturbed"); + var requestFromRequest = new Request(bodyRequest); + assert_true(bodyRequest.bodyUsed , "bodyUsed is true when request is disturbed"); + assert_equals(bodyRequest.body, originalBody, "body should not change"); + assert_not_equals(originalBody, undefined, "body should not be undefined"); + assert_not_equals(originalBody, null, "body should not be null"); + assert_not_equals(requestFromRequest.body, originalBody, "new request's body is new"); + return requestFromRequest.text().then(text => { + assert_equals(text, "Request's body"); + }); +}, "Input request used for creating new request became disturbed"); + +promise_test(() => { + const bodyRequest = new Request("", initValuesDict); + const originalBody = bodyRequest.body; + assert_false(bodyRequest.bodyUsed , "bodyUsed is false when request is not disturbed"); + const requestFromRequest = new Request(bodyRequest, { body : "init body" }); + assert_true(bodyRequest.bodyUsed , "bodyUsed is true when request is disturbed"); + assert_equals(bodyRequest.body, originalBody, "body should not change"); + assert_not_equals(originalBody, undefined, "body should not be undefined"); + assert_not_equals(originalBody, null, "body should not be null"); + assert_not_equals(requestFromRequest.body, originalBody, "new request's body is new"); + + return requestFromRequest.text().then(text => { + assert_equals(text, "init body"); + }); +}, "Input request used for creating new request became disturbed even if body is not used"); + +promise_test(function(test) { + assert_true(bodyConsumed.bodyUsed , "bodyUsed is true when request is disturbed"); + return promise_rejects_js(test, TypeError, bodyConsumed.blob()); +}, "Check consuming a disturbed request"); + +test(function() { + var req = new Request(URL, {method: 'POST', body: 'hello'}); + assert_false(req.bodyUsed, + 'Request should not be flagged as used if it has not been ' + + 'consumed.'); + assert_throws_js(TypeError, + function() { new Request(req, {method: 'GET'}); }, + 'A get request may not have body.'); + + assert_false(req.bodyUsed, 'After the GET case'); + + assert_throws_js(TypeError, + function() { new Request(req, {method: 'CONNECT'}); }, + 'Request() with a forbidden method must throw.'); + + assert_false(req.bodyUsed, 'After the forbidden method case'); + + var req2 = new Request(req); + assert_true(req.bodyUsed, + 'Request should be flagged as used if it has been consumed.'); +}, 'Request construction failure should not set "bodyUsed"'); diff --git a/pkg/download/engine/testdata/wpt/fetch/api/request/request-error.any.js b/pkg/download/engine/testdata/wpt/fetch/api/request/request-error.any.js new file mode 100644 index 00000000..9ec80151 --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/request/request-error.any.js @@ -0,0 +1,56 @@ +// META: global=window,worker +// META: title=Request error +// META: script=request-error.js + +// badRequestArgTests is from response-error.js +for (const { args, testName } of badRequestArgTests) { + test(() => { + assert_throws_js( + TypeError, + () => new Request(...args), + "Expect TypeError exception" + ); + }, testName); +} + +test(function() { + assert_throws_js( + TypeError, + () => Request("about:blank"), + "Calling Request constructor without 'new' must throw" + ); +}); + +test(function() { + var initialHeaders = new Headers([["Content-Type", "potato"]]); + var initialRequest = new Request("", {"headers" : initialHeaders}); + var request = new Request(initialRequest); + assert_equals(request.headers.get("Content-Type"), "potato"); +}, "Request should get its content-type from the init request"); + +test(function() { + var initialHeaders = new Headers([["Content-Type", "potato"]]); + var initialRequest = new Request("", {"headers" : initialHeaders}); + var headers = new Headers([]); + var request = new Request(initialRequest, {"headers" : headers}); + assert_false(request.headers.has("Content-Type")); +}, "Request should not get its content-type from the init request if init headers are provided"); + +test(function() { + var initialHeaders = new Headers([["Content-Type-Extra", "potato"]]); + var initialRequest = new Request("", {"headers" : initialHeaders, "body" : "this is my plate", "method" : "POST"}); + var request = new Request(initialRequest); + assert_equals(request.headers.get("Content-Type"), "text/plain;charset=UTF-8"); +}, "Request should get its content-type from the body if none is provided"); + +test(function() { + var initialHeaders = new Headers([["Content-Type", "potato"]]); + var initialRequest = new Request("", {"headers" : initialHeaders, "body" : "this is my plate", "method" : "POST"}); + var request = new Request(initialRequest); + assert_equals(request.headers.get("Content-Type"), "potato"); +}, "Request should get its content-type from init headers if one is provided"); + +test(function() { + var options = {"cache": "only-if-cached", "mode": "same-origin"}; + new Request("test", options); +}, "Request with cache mode: only-if-cached and fetch mode: same-origin"); diff --git a/pkg/download/engine/testdata/wpt/fetch/api/request/request-error.js b/pkg/download/engine/testdata/wpt/fetch/api/request/request-error.js new file mode 100644 index 00000000..cf77313f --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/request/request-error.js @@ -0,0 +1,57 @@ +const badRequestArgTests = [ + { + args: ["", { "window": "http://test.url" }], + testName: "RequestInit's window is not null" + }, + { + args: ["http://:not a valid URL"], + testName: "Input URL is not valid" + }, + { + args: ["http://user:pass@test.url"], + testName: "Input URL has credentials" + }, + { + args: ["", { "mode": "navigate" }], + testName: "RequestInit's mode is navigate" + }, + { + args: ["", { "referrer": "http://:not a valid URL" }], + testName: "RequestInit's referrer is invalid" + }, + { + args: ["", { "method": "IN VALID" }], + testName: "RequestInit's method is invalid" + }, + { + args: ["", { "method": "TRACE" }], + testName: "RequestInit's method is forbidden" + }, + { + args: ["", { "mode": "no-cors", "method": "PUT" }], + testName: "RequestInit's mode is no-cors and method is not simple" + }, + { + args: ["", { "mode": "cors", "cache": "only-if-cached" }], + testName: "RequestInit's cache mode is only-if-cached and mode is not same-origin" + }, + { + args: ["test", { "cache": "only-if-cached", "mode": "cors" }], + testName: "Request with cache mode: only-if-cached and fetch mode cors" + }, + { + args: ["test", { "cache": "only-if-cached", "mode": "no-cors" }], + testName: "Request with cache mode: only-if-cached and fetch mode no-cors" + } +]; + +badRequestArgTests.push( + ...["referrerPolicy", "mode", "credentials", "cache", "redirect"].map(optionProp => { + const options = {}; + options[optionProp] = "BAD"; + return { + args: ["", options], + testName: `Bad ${optionProp} init parameter value` + }; + }) +); diff --git a/pkg/download/engine/testdata/wpt/fetch/api/request/request-headers.any.js b/pkg/download/engine/testdata/wpt/fetch/api/request/request-headers.any.js new file mode 100644 index 00000000..a766bcb5 --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/request/request-headers.any.js @@ -0,0 +1,177 @@ +// META: global=window,worker +// META: title=Request Headers + +var validRequestHeaders = [ + ["Content-Type", "OK"], + ["Potato", "OK"], + ["proxy", "OK"], + ["proxya", "OK"], + ["sec", "OK"], + ["secb", "OK"], + ["Set-Cookie2", "OK"], + ["User-Agent", "OK"], +]; +var invalidRequestHeaders = [ + ["Accept-Charset", "KO"], + ["accept-charset", "KO"], + ["ACCEPT-ENCODING", "KO"], + ["Accept-Encoding", "KO"], + ["Access-Control-Request-Headers", "KO"], + ["Access-Control-Request-Method", "KO"], + ["Connection", "KO"], + ["Content-Length", "KO"], + ["Cookie", "KO"], + ["Cookie2", "KO"], + ["Date", "KO"], + ["DNT", "KO"], + ["Expect", "KO"], + ["Host", "KO"], + ["Keep-Alive", "KO"], + ["Origin", "KO"], + ["Referer", "KO"], + ["Set-Cookie", "KO"], + ["TE", "KO"], + ["Trailer", "KO"], + ["Transfer-Encoding", "KO"], + ["Upgrade", "KO"], + ["Via", "KO"], + ["Proxy-", "KO"], + ["proxy-a", "KO"], + ["Sec-", "KO"], + ["sec-b", "KO"], +]; + +var validRequestNoCorsHeaders = [ + ["Accept", "OK"], + ["Accept-Language", "OK"], + ["content-language", "OK"], + ["content-type", "application/x-www-form-urlencoded"], + ["content-type", "application/x-www-form-urlencoded;charset=UTF-8"], + ["content-type", "multipart/form-data"], + ["content-type", "multipart/form-data;charset=UTF-8"], + ["content-TYPE", "text/plain"], + ["CONTENT-type", "text/plain;charset=UTF-8"], +]; +var invalidRequestNoCorsHeaders = [ + ["Content-Type", "KO"], + ["Potato", "KO"], + ["proxy", "KO"], + ["proxya", "KO"], + ["sec", "KO"], + ["secb", "KO"], + ["Empty-Value", ""], +]; + +validRequestHeaders.forEach(function(header) { + test(function() { + var request = new Request(""); + request.headers.set(header[0], header[1]); + assert_equals(request.headers.get(header[0]), header[1]); + }, "Adding valid request header \"" + header[0] + ": " + header[1] + "\""); +}); +invalidRequestHeaders.forEach(function(header) { + test(function() { + var request = new Request(""); + request.headers.set(header[0], header[1]); + assert_equals(request.headers.get(header[0]), null); + }, "Adding invalid request header \"" + header[0] + ": " + header[1] + "\""); +}); + +validRequestNoCorsHeaders.forEach(function(header) { + test(function() { + var requestNoCors = new Request("", {"mode": "no-cors"}); + requestNoCors.headers.set(header[0], header[1]); + assert_equals(requestNoCors.headers.get(header[0]), header[1]); + }, "Adding valid no-cors request header \"" + header[0] + ": " + header[1] + "\""); +}); +invalidRequestNoCorsHeaders.forEach(function(header) { + test(function() { + var requestNoCors = new Request("", {"mode": "no-cors"}); + requestNoCors.headers.set(header[0], header[1]); + assert_equals(requestNoCors.headers.get(header[0]), null); + }, "Adding invalid no-cors request header \"" + header[0] + ": " + header[1] + "\""); +}); + +test(function() { + var headers = new Headers([["Cookie2", "potato"]]); + var request = new Request("", {"headers": headers}); + assert_equals(request.headers.get("Cookie2"), null); +}, "Check that request constructor is filtering headers provided as init parameter"); + +test(function() { + var headers = new Headers([["Content-Type", "potato"]]); + var request = new Request("", {"headers": headers, "mode": "no-cors"}); + assert_equals(request.headers.get("Content-Type"), null); +}, "Check that no-cors request constructor is filtering headers provided as init parameter"); + +test(function() { + var headers = new Headers([["Content-Type", "potato"]]); + var initialRequest = new Request("", {"headers": headers}); + var request = new Request(initialRequest, {"mode": "no-cors"}); + assert_equals(request.headers.get("Content-Type"), null); +}, "Check that no-cors request constructor is filtering headers provided as part of request parameter"); + +test(function() { + var initialHeaders = new Headers([["Content-Type", "potato"]]); + var initialRequest = new Request("", {"headers" : initialHeaders}); + var request = new Request(initialRequest); + assert_equals(request.headers.get("Content-Type"), "potato"); +}, "Request should get its content-type from the init request"); + +test(function() { + var initialHeaders = new Headers([["Content-Type", "potato"]]); + var initialRequest = new Request("", {"headers" : initialHeaders}); + var headers = new Headers([]); + var request = new Request(initialRequest, {"headers" : headers}); + assert_false(request.headers.has("Content-Type")); +}, "Request should not get its content-type from the init request if init headers are provided"); + +test(function() { + var initialHeaders = new Headers([["Content-Type-Extra", "potato"]]); + var initialRequest = new Request("", {"headers" : initialHeaders, "body" : "this is my plate", "method" : "POST"}); + var request = new Request(initialRequest); + assert_equals(request.headers.get("Content-Type"), "text/plain;charset=UTF-8"); +}, "Request should get its content-type from the body if none is provided"); + +test(function() { + var initialHeaders = new Headers([["Content-Type", "potato"]]); + var initialRequest = new Request("", {"headers" : initialHeaders, "body" : "this is my plate", "method" : "POST"}); + var request = new Request(initialRequest); + assert_equals(request.headers.get("Content-Type"), "potato"); +}, "Request should get its content-type from init headers if one is provided"); + +test(function() { + var array = [["hello", "worldAHH"]]; + var object = {"hello": 'worldOOH'}; + var headers = new Headers(array); + + assert_equals(headers.get("hello"), "worldAHH"); + + var request1 = new Request("", {"headers": headers}); + var request2 = new Request("", {"headers": array}); + var request3 = new Request("", {"headers": object}); + + assert_equals(request1.headers.get("hello"), "worldAHH"); + assert_equals(request2.headers.get("hello"), "worldAHH"); + assert_equals(request3.headers.get("hello"), "worldOOH"); +}, "Testing request header creations with various objects"); + +promise_test(function(test) { + var request = new Request("", {"headers" : [["Content-Type", ""]], "body" : "this is my plate", "method" : "POST"}); + return request.blob().then(function(blob) { + assert_equals(blob.type, "", "Blob type should be the empty string"); + }); +}, "Testing empty Request Content-Type header"); + +test(function() { + const request1 = new Request(""); + assert_equals(request1.headers, request1.headers); + + const request2 = new Request("", {"headers": {"X-Foo": "bar"}}); + assert_equals(request2.headers, request2.headers); + const headers = request2.headers; + request2.headers.set("X-Foo", "quux"); + assert_equals(headers, request2.headers); + headers.set("X-Other-Header", "baz"); + assert_equals(headers, request2.headers); +}, "Test that Request.headers has the [SameObject] extended attribute"); diff --git a/pkg/download/engine/testdata/wpt/fetch/api/request/request-init-002.any.js b/pkg/download/engine/testdata/wpt/fetch/api/request/request-init-002.any.js new file mode 100644 index 00000000..abb6689f --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/request/request-init-002.any.js @@ -0,0 +1,60 @@ +// META: global=window,worker +// META: title=Request init: headers and body + +test(function() { + var headerDict = {"name1": "value1", + "name2": "value2", + "name3": "value3" + }; + var headers = new Headers(headerDict); + var request = new Request("", { "headers" : headers }) + for (var name in headerDict) { + assert_equals(request.headers.get(name), headerDict[name], + "request's headers has " + name + " : " + headerDict[name]); + } +}, "Initialize Request with headers values"); + +function makeRequestInit(body, method) { + return {"method": method, "body": body}; +} + +function checkRequestInit(body, bodyType, expectedTextBody) { + promise_test(function(test) { + var request = new Request("", makeRequestInit(body, "POST")); + if (body) { + assert_throws_js(TypeError, function() { new Request("", makeRequestInit(body, "GET")); }); + assert_throws_js(TypeError, function() { new Request("", makeRequestInit(body, "HEAD")); }); + } else { + new Request("", makeRequestInit(body, "GET")); // should not throw + } + var reqHeaders = request.headers; + var mime = reqHeaders.get("Content-Type"); + assert_true(!body || (mime && mime.search(bodyType) > -1), "Content-Type header should be \"" + bodyType + "\", not \"" + mime + "\""); + return request.text().then(function(bodyAsText) { + //not equals: cannot guess formData exact value + assert_true( bodyAsText.search(expectedTextBody) > -1, "Retrieve and verify request body"); + }); + }, `Initialize Request's body with "${body}", ${bodyType}`); +} + +var blob = new Blob(["This is a blob"], {type: "application/octet-binary"}); +var formaData = new FormData(); +formaData.append("name", "value"); +var usvString = "This is a USVString" + +checkRequestInit(undefined, undefined, ""); +checkRequestInit(null, null, ""); +checkRequestInit(blob, "application/octet-binary", "This is a blob"); +checkRequestInit(formaData, "multipart/form-data", "name=\"name\"\r\n\r\nvalue"); +checkRequestInit(usvString, "text/plain;charset=UTF-8", "This is a USVString"); +checkRequestInit({toString: () => "hi!"}, "text/plain;charset=UTF-8", "hi!"); + +// Ensure test does not time out in case of missing URLSearchParams support. +if (self.URLSearchParams) { + var urlSearchParams = new URLSearchParams("name=value"); + checkRequestInit(urlSearchParams, "application/x-www-form-urlencoded;charset=UTF-8", "name=value"); +} else { + promise_test(function(test) { + return Promise.reject("URLSearchParams not supported"); + }, "Initialize Request's body with application/x-www-form-urlencoded;charset=UTF-8"); +} diff --git a/pkg/download/engine/testdata/wpt/fetch/api/request/request-init-contenttype.any.js b/pkg/download/engine/testdata/wpt/fetch/api/request/request-init-contenttype.any.js new file mode 100644 index 00000000..18a6969d --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/request/request-init-contenttype.any.js @@ -0,0 +1,141 @@ +function requestFromBody(body) { + return new Request( + "https://example.com", + { + method: "POST", + body, + duplex: "half", + }, + ); +} + +test(() => { + const request = requestFromBody(undefined); + assert_equals(request.headers.get("Content-Type"), null); +}, "Default Content-Type for Request with empty body"); + +test(() => { + const blob = new Blob([]); + const request = requestFromBody(blob); + assert_equals(request.headers.get("Content-Type"), null); +}, "Default Content-Type for Request with Blob body (no type set)"); + +test(() => { + const blob = new Blob([], { type: "" }); + const request = requestFromBody(blob); + assert_equals(request.headers.get("Content-Type"), null); +}, "Default Content-Type for Request with Blob body (empty type)"); + +test(() => { + const blob = new Blob([], { type: "a/b; c=d" }); + const request = requestFromBody(blob); + assert_equals(request.headers.get("Content-Type"), "a/b; c=d"); +}, "Default Content-Type for Request with Blob body (set type)"); + +test(() => { + const buffer = new Uint8Array(); + const request = requestFromBody(buffer); + assert_equals(request.headers.get("Content-Type"), null); +}, "Default Content-Type for Request with buffer source body"); + +promise_test(async () => { + const formData = new FormData(); + formData.append("a", "b"); + const request = requestFromBody(formData); + const boundary = (await request.text()).split("\r\n")[0].slice(2); + assert_equals( + request.headers.get("Content-Type"), + `multipart/form-data; boundary=${boundary}`, + ); +}, "Default Content-Type for Request with FormData body"); + +test(() => { + const usp = new URLSearchParams(); + const request = requestFromBody(usp); + assert_equals( + request.headers.get("Content-Type"), + "application/x-www-form-urlencoded;charset=UTF-8", + ); +}, "Default Content-Type for Request with URLSearchParams body"); + +test(() => { + const request = requestFromBody(""); + assert_equals( + request.headers.get("Content-Type"), + "text/plain;charset=UTF-8", + ); +}, "Default Content-Type for Request with string body"); + +test(() => { + const stream = new ReadableStream(); + const request = requestFromBody(stream); + assert_equals(request.headers.get("Content-Type"), null); +}, "Default Content-Type for Request with ReadableStream body"); + +// ----------------------------------------------------------------------------- + +const OVERRIDE_MIME = "test/only; mime=type"; + +function requestFromBodyWithOverrideMime(body) { + return new Request( + "https://example.com", + { + method: "POST", + body, + headers: { "Content-Type": OVERRIDE_MIME }, + duplex: "half", + }, + ); +} + +test(() => { + const request = requestFromBodyWithOverrideMime(undefined); + assert_equals(request.headers.get("Content-Type"), OVERRIDE_MIME); +}, "Can override Content-Type for Request with empty body"); + +test(() => { + const blob = new Blob([]); + const request = requestFromBodyWithOverrideMime(blob); + assert_equals(request.headers.get("Content-Type"), OVERRIDE_MIME); +}, "Can override Content-Type for Request with Blob body (no type set)"); + +test(() => { + const blob = new Blob([], { type: "" }); + const request = requestFromBodyWithOverrideMime(blob); + assert_equals(request.headers.get("Content-Type"), OVERRIDE_MIME); +}, "Can override Content-Type for Request with Blob body (empty type)"); + +test(() => { + const blob = new Blob([], { type: "a/b; c=d" }); + const request = requestFromBodyWithOverrideMime(blob); + assert_equals(request.headers.get("Content-Type"), OVERRIDE_MIME); +}, "Can override Content-Type for Request with Blob body (set type)"); + +test(() => { + const buffer = new Uint8Array(); + const request = requestFromBodyWithOverrideMime(buffer); + assert_equals(request.headers.get("Content-Type"), OVERRIDE_MIME); +}, "Can override Content-Type for Request with buffer source body"); + +test(() => { + const formData = new FormData(); + const request = requestFromBodyWithOverrideMime(formData); + assert_equals(request.headers.get("Content-Type"), OVERRIDE_MIME); +}, "Can override Content-Type for Request with FormData body"); + +test(() => { + const usp = new URLSearchParams(); + const request = requestFromBodyWithOverrideMime(usp); + assert_equals(request.headers.get("Content-Type"), OVERRIDE_MIME); +}, "Can override Content-Type for Request with URLSearchParams body"); + +test(() => { + const request = requestFromBodyWithOverrideMime(""); + assert_equals(request.headers.get("Content-Type"), OVERRIDE_MIME); +}, "Can override Content-Type for Request with string body"); + +test(() => { + const stream = new ReadableStream(); + const request = requestFromBodyWithOverrideMime(stream); + assert_equals(request.headers.get("Content-Type"), OVERRIDE_MIME); +}, "Can override Content-Type for Request with ReadableStream body"); diff --git a/pkg/download/engine/testdata/wpt/fetch/api/request/request-init-priority.any.js b/pkg/download/engine/testdata/wpt/fetch/api/request/request-init-priority.any.js new file mode 100644 index 00000000..eb5073c8 --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/request/request-init-priority.any.js @@ -0,0 +1,26 @@ +var priorities = ["high", + "low", + "auto" + ]; + +for (idx in priorities) { + test(() => { + new Request("", {priority: priorities[idx]}); + }, "new Request() with a '" + priorities[idx] + "' priority does not throw an error"); +} + +test(() => { + assert_throws_js(TypeError, () => { + new Request("", {priority: 'invalid'}); + }, "a new Request() must throw a TypeError if RequestInit's priority is an invalid value"); +}, "new Request() throws a TypeError if any of RequestInit's members' values are invalid"); + +for (idx in priorities) { + promise_test(function(t) { + return fetch('hello.txt', { priority: priorities[idx] }); + }, "fetch() with a '" + priorities[idx] + "' priority completes successfully"); +} + +promise_test(function(t) { + return promise_rejects_js(t, TypeError, fetch('hello.txt', { priority: 'invalid' })); +}, "fetch() with an invalid priority returns a rejected promise with a TypeError"); diff --git a/pkg/download/engine/testdata/wpt/fetch/api/request/request-init-stream.any.js b/pkg/download/engine/testdata/wpt/fetch/api/request/request-init-stream.any.js new file mode 100644 index 00000000..43f8509d --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/request/request-init-stream.any.js @@ -0,0 +1,146 @@ +// META: global=window,worker + +"use strict"; + +const duplex = "half"; +const method = "POST"; + +test(() => { + const body = new ReadableStream(); + const request = new Request("...", { method, body, duplex }); + assert_equals(request.body, body); +}, "Constructing a Request with a stream holds the original object."); + +test((t) => { + const body = new ReadableStream(); + body.getReader(); + assert_throws_js(TypeError, + () => new Request("...", { method, body, duplex })); +}, "Constructing a Request with a stream on which getReader() is called"); + +test((t) => { + const body = new ReadableStream(); + body.getReader().read(); + assert_throws_js(TypeError, + () => new Request("...", { method, body, duplex })); +}, "Constructing a Request with a stream on which read() is called"); + +promise_test(async (t) => { + const body = new ReadableStream({ pull: c => c.enqueue(new Uint8Array()) }); + const reader = body.getReader(); + await reader.read(); + reader.releaseLock(); + assert_throws_js(TypeError, + () => new Request("...", { method, body, duplex })); +}, "Constructing a Request with a stream on which read() and releaseLock() are called"); + +test((t) => { + const request = new Request("...", { method: "POST", body: "..." }); + request.body.getReader(); + assert_throws_js(TypeError, () => new Request(request)); + // This doesn't throw. + new Request(request, { body: "..." }); +}, "Constructing a Request with a Request on which body.getReader() is called"); + +test((t) => { + const request = new Request("...", { method: "POST", body: "..." }); + request.body.getReader().read(); + assert_throws_js(TypeError, () => new Request(request)); + // This doesn't throw. + new Request(request, { body: "..." }); +}, "Constructing a Request with a Request on which body.getReader().read() is called"); + +promise_test(async (t) => { + const request = new Request("...", { method: "POST", body: "..." }); + const reader = request.body.getReader(); + await reader.read(); + reader.releaseLock(); + assert_throws_js(TypeError, () => new Request(request)); + // This doesn't throw. + new Request(request, { body: "..." }); +}, "Constructing a Request with a Request on which read() and releaseLock() are called"); + +test((t) => { + new Request("...", { method, body: null }); +}, "It is OK to omit .duplex when the body is null."); + +test((t) => { + new Request("...", { method, body: "..." }); +}, "It is OK to omit .duplex when the body is a string."); + +test((t) => { + new Request("...", { method, body: new Uint8Array(3) }); +}, "It is OK to omit .duplex when the body is a Uint8Array."); + +test((t) => { + new Request("...", { method, body: new Blob([]) }); +}, "It is OK to omit .duplex when the body is a Blob."); + +test((t) => { + const body = new ReadableStream(); + assert_throws_js(TypeError, + () => new Request("...", { method, body })); +}, "It is error to omit .duplex when the body is a ReadableStream."); + +test((t) => { + new Request("...", { method, body: null, duplex: "half" }); +}, "It is OK to set .duplex = 'half' when the body is null."); + +test((t) => { + new Request("...", { method, body: "...", duplex: "half" }); +}, "It is OK to set .duplex = 'half' when the body is a string."); + +test((t) => { + new Request("...", { method, body: new Uint8Array(3), duplex: "half" }); +}, "It is OK to set .duplex = 'half' when the body is a Uint8Array."); + +test((t) => { + new Request("...", { method, body: new Blob([]), duplex: "half" }); +}, "It is OK to set .duplex = 'half' when the body is a Blob."); + +test((t) => { + const body = new ReadableStream(); + new Request("...", { method, body, duplex: "half" }); +}, "It is OK to set .duplex = 'half' when the body is a ReadableStream."); + +test((t) => { + const body = null; + const duplex = "full"; + assert_throws_js(TypeError, + () => new Request("...", { method, body, duplex })); +}, "It is error to set .duplex = 'full' when the body is null."); + +test((t) => { + const body = "..."; + const duplex = "full"; + assert_throws_js(TypeError, + () => new Request("...", { method, body, duplex })); +}, "It is error to set .duplex = 'full' when the body is a string."); + +test((t) => { + const body = new Uint8Array(3); + const duplex = "full"; + assert_throws_js(TypeError, + () => new Request("...", { method, body, duplex })); +}, "It is error to set .duplex = 'full' when the body is a Uint8Array."); + +test((t) => { + const body = new Blob([]); + const duplex = "full"; + assert_throws_js(TypeError, + () => new Request("...", { method, body, duplex })); +}, "It is error to set .duplex = 'full' when the body is a Blob."); + +test((t) => { + const body = new ReadableStream(); + const duplex = "full"; + assert_throws_js(TypeError, + () => new Request("...", { method, body, duplex })); +}, "It is error to set .duplex = 'full' when the body is a ReadableStream."); + +test((t) => { + const body = new ReadableStream(); + const duplex = "half"; + const req1 = new Request("...", { method, body, duplex }); + const req2 = new Request(req1); +}, "It is OK to omit duplex when init.body is not given and input.body is given."); diff --git a/pkg/download/engine/testdata/wpt/fetch/api/request/request-keepalive.any.js b/pkg/download/engine/testdata/wpt/fetch/api/request/request-keepalive.any.js new file mode 100644 index 00000000..cb4506db --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/request/request-keepalive.any.js @@ -0,0 +1,17 @@ +// META: global=window,worker +// META: title=Request keepalive +// META: script=/common/utils.js +// META: script=/common/get-host-info.sub.js + +test(() => { + assert_false(new Request('/').keepalive, 'default'); + assert_true(new Request('/', {keepalive: true}).keepalive, 'true'); + assert_false(new Request('/', {keepalive: false}).keepalive, 'false'); + assert_true(new Request('/', {keepalive: 1}).keepalive, 'truish'); + assert_false(new Request('/', {keepalive: 0}).keepalive, 'falsy'); +}, 'keepalive flag'); + +test(() => { + const init = {method: 'POST', keepalive: true, body: new ReadableStream()}; + assert_throws_js(TypeError, () => {new Request('/', init)}); +}, 'keepalive flag with stream body'); diff --git a/pkg/download/engine/testdata/wpt/fetch/api/request/request-structure.any.js b/pkg/download/engine/testdata/wpt/fetch/api/request/request-structure.any.js new file mode 100644 index 00000000..70a7c628 --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/request/request-structure.any.js @@ -0,0 +1,143 @@ +// META: global=window,worker +// META: title=Request structure + +var request = new Request(""); +var methods = ["clone", + //Request implements Body + "arrayBuffer", + "blob", + "formData", + "json", + "text" + ]; +var attributes = ["method", + "url", + "headers", + "destination", + "referrer", + "referrerPolicy", + "mode", + "credentials", + "cache", + "redirect", + "integrity", + "isReloadNavigation", + "isHistoryNavigation", + "duplex", + //Request implements Body + "bodyUsed" + ]; +var internalAttributes = ["priority", + "internalpriority", + "blocking" + ]; + +function isReadOnly(request, attributeToCheck) { + var defaultValue = undefined; + var newValue = undefined; + switch (attributeToCheck) { + case "method": + defaultValue = "GET"; + newValue = "POST"; + break; + + case "url": + //default value is base url + //i.e http://example.com/fetch/api/request-structure.html + newValue = "http://url.test"; + break; + + case "headers": + request.headers = new Headers ({"name":"value"}); + assert_false(request.headers.has("name"), "Headers attribute is read only"); + return; + + case "destination": + defaultValue = ""; + newValue = "worker"; + break; + + case "referrer": + defaultValue = "about:client"; + newValue = "http://url.test"; + break; + + case "referrerPolicy": + defaultValue = ""; + newValue = "unsafe-url"; + break; + + case "mode": + defaultValue = "cors"; + newValue = "navigate"; + break; + + case "credentials": + defaultValue = "same-origin"; + newValue = "cors"; + break; + + case "cache": + defaultValue = "default"; + newValue = "reload"; + break; + + case "redirect": + defaultValue = "follow"; + newValue = "manual"; + break; + + case "integrity": + newValue = "CannotWriteIntegrity"; + break; + + case "bodyUsed": + defaultValue = false; + newValue = true; + break; + + case "isReloadNavigation": + defaultValue = false; + newValue = true; + break; + + case "isHistoryNavigation": + defaultValue = false; + newValue = true; + break; + + case "duplex": + defaultValue = "half"; + newValue = "full"; + break; + + default: + return; + } + + request[attributeToCheck] = newValue; + if (defaultValue === undefined) + assert_not_equals(request[attributeToCheck], newValue, "Attribute " + attributeToCheck + " is read only"); + else + assert_equals(request[attributeToCheck], defaultValue, + "Attribute " + attributeToCheck + " is read only. Default value is " + defaultValue); +} + +for (var idx in methods) { + test(function() { + assert_true(methods[idx] in request, "request has " + methods[idx] + " method"); + }, "Request has " + methods[idx] + " method"); +} + +for (var idx in attributes) { + test(function() { + assert_true(attributes[idx] in request, "request has " + attributes[idx] + " attribute"); + isReadOnly(request, attributes[idx]); + }, "Check " + attributes[idx] + " attribute"); +} + +for (var idx in internalAttributes) { + test(function() { + assert_false(internalAttributes[idx] in request, "request does not expose " + internalAttributes[idx] + " attribute"); + }, "Request does not expose " + internalAttributes[idx] + " attribute"); +} diff --git a/pkg/download/engine/testdata/wpt/fetch/api/resources/utils.js b/pkg/download/engine/testdata/wpt/fetch/api/resources/utils.js new file mode 100644 index 00000000..3721d9bf --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/resources/utils.js @@ -0,0 +1,120 @@ +var RESOURCES_DIR = "../resources/"; + +function dirname(path) { + return path.replace(/\/[^\/]*$/, '/') +} + +function checkRequest(request, ExpectedValuesDict) { + for (var attribute in ExpectedValuesDict) { + switch(attribute) { + case "headers": + for (var key in ExpectedValuesDict["headers"].keys()) { + assert_equals(request["headers"].get(key), ExpectedValuesDict["headers"].get(key), + "Check headers attribute has " + key + ":" + ExpectedValuesDict["headers"].get(key)); + } + break; + + case "body": + //for checking body's content, a dedicated asyncronous/promise test should be used + assert_true(request["headers"].has("Content-Type") , "Check request has body using Content-Type header") + break; + + case "method": + case "referrer": + case "referrerPolicy": + case "credentials": + case "cache": + case "redirect": + case "integrity": + case "url": + case "destination": + assert_equals(request[attribute], ExpectedValuesDict[attribute], "Check " + attribute + " attribute") + break; + + default: + break; + } + } +} + +function stringToArray(str) { + var array = new Uint8Array(str.length); + for (var i=0, strLen = str.length; i < strLen; i++) + array[i] = str.charCodeAt(i); + return array; +} + +function encode_utf8(str) +{ + if (self.TextEncoder) + return (new TextEncoder).encode(str); + return stringToArray(unescape(encodeURIComponent(str))); +} + +function validateBufferFromString(buffer, expectedValue, message) +{ + return assert_array_equals(new Uint8Array(buffer !== undefined ? buffer : []), stringToArray(expectedValue), message); +} + +function validateStreamFromString(reader, expectedValue, retrievedArrayBuffer) { + // Passing Uint8Array for byte streams; non-byte streams will simply ignore it + return reader.read(new Uint8Array(64)).then(function(data) { + if (!data.done) { + assert_true(data.value instanceof Uint8Array, "Fetch ReadableStream chunks should be Uint8Array"); + var newBuffer; + if (retrievedArrayBuffer) { + newBuffer = new Uint8Array(data.value.length + retrievedArrayBuffer.length); + newBuffer.set(retrievedArrayBuffer, 0); + newBuffer.set(data.value, retrievedArrayBuffer.length); + } else { + newBuffer = data.value; + } + return validateStreamFromString(reader, expectedValue, newBuffer); + } + validateBufferFromString(retrievedArrayBuffer, expectedValue, "Retrieve and verify stream"); + }); +} + +function validateStreamFromPartialString(reader, expectedValue, retrievedArrayBuffer) { + // Passing Uint8Array for byte streams; non-byte streams will simply ignore it + return reader.read(new Uint8Array(64)).then(function(data) { + if (!data.done) { + assert_true(data.value instanceof Uint8Array, "Fetch ReadableStream chunks should be Uint8Array"); + var newBuffer; + if (retrievedArrayBuffer) { + newBuffer = new Uint8Array(data.value.length + retrievedArrayBuffer.length); + newBuffer.set(retrievedArrayBuffer, 0); + newBuffer.set(data.value, retrievedArrayBuffer.length); + } else { + newBuffer = data.value; + } + return validateStreamFromPartialString(reader, expectedValue, newBuffer); + } + + var string = new TextDecoder("utf-8").decode(retrievedArrayBuffer); + return assert_true(string.search(expectedValue) != -1, "Retrieve and verify stream"); + }); +} + +// From streams tests +function delay(milliseconds) +{ + return new Promise(function(resolve) { + step_timeout(resolve, milliseconds); + }); +} + +function requestForbiddenHeaders(desc, forbiddenHeaders) { + var url = RESOURCES_DIR + "inspect-headers.py"; + var requestInit = {"headers": forbiddenHeaders} + var urlParameters = "?headers=" + Object.keys(forbiddenHeaders).join("|"); + + promise_test(function(test){ + return fetch(url + urlParameters, requestInit).then(function(resp) { + assert_equals(resp.status, 200, "HTTP status is 200"); + assert_equals(resp.type , "basic", "Response's type is basic"); + for (var header in forbiddenHeaders) + assert_not_equals(resp.headers.get("x-request-" + header), forbiddenHeaders[header], header + " does not have the value we defined"); + }); + }, desc); +} diff --git a/pkg/download/engine/testdata/wpt/fetch/api/response/response-consume-empty.any.js b/pkg/download/engine/testdata/wpt/fetch/api/response/response-consume-empty.any.js new file mode 100644 index 00000000..a5df3562 --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/response/response-consume-empty.any.js @@ -0,0 +1,88 @@ +// META: global=window,worker +// META: title=Response consume empty bodies + +function checkBodyText(test, response) { + return response.text().then(function(bodyAsText) { + assert_equals(bodyAsText, "", "Resolved value should be empty"); + assert_false(response.bodyUsed); + }); +} + +async function checkBodyBlob(test, response) { + const bodyAsBlob = await response.blob(); + const body = await bodyAsBlob.text(); + + assert_equals(body, "", "Resolved value should be empty"); + assert_false(response.bodyUsed); +} + +function checkBodyArrayBuffer(test, response) { + return response.arrayBuffer().then(function(bodyAsArrayBuffer) { + assert_equals(bodyAsArrayBuffer.byteLength, 0, "Resolved value should be empty"); + assert_false(response.bodyUsed); + }); +} + +function checkBodyJSON(test, response) { + return response.json().then( + function(bodyAsJSON) { + assert_unreached("JSON parsing should fail"); + }, + function() { + assert_false(response.bodyUsed); + }); +} + +function checkBodyFormData(test, response) { + return response.formData().then(function(bodyAsFormData) { + assert_true(bodyAsFormData instanceof FormData, "Should receive a FormData"); + assert_false(response.bodyUsed); + }); +} + +function checkBodyFormDataError(test, response) { + return promise_rejects_js(test, TypeError, response.formData()).then(function() { + assert_false(response.bodyUsed); + }); +} + +function checkResponseWithNoBody(bodyType, checkFunction, headers = []) { + promise_test(function(test) { + var response = new Response(undefined, { "headers": headers }); + assert_false(response.bodyUsed); + return checkFunction(test, response); + }, "Consume response's body as " + bodyType); +} + +checkResponseWithNoBody("text", checkBodyText); +checkResponseWithNoBody("blob", checkBodyBlob); +checkResponseWithNoBody("arrayBuffer", checkBodyArrayBuffer); +checkResponseWithNoBody("json (error case)", checkBodyJSON); +checkResponseWithNoBody("formData with correct multipart type (error case)", checkBodyFormDataError, [["Content-Type", 'multipart/form-data; boundary="boundary"']]); +checkResponseWithNoBody("formData with correct urlencoded type", checkBodyFormData, [["Content-Type", "application/x-www-form-urlencoded;charset=UTF-8"]]); +checkResponseWithNoBody("formData without correct type (error case)", checkBodyFormDataError); + +function checkResponseWithEmptyBody(bodyType, body, asText) { + promise_test(function(test) { + var response = new Response(body); + assert_false(response.bodyUsed, "bodyUsed is false at init"); + if (asText) { + return response.text().then(function(bodyAsString) { + assert_equals(bodyAsString.length, 0, "Resolved value should be empty"); + assert_true(response.bodyUsed, "bodyUsed is true after being consumed"); + }); + } + return response.arrayBuffer().then(function(bodyAsArrayBuffer) { + assert_equals(bodyAsArrayBuffer.byteLength, 0, "Resolved value should be empty"); + assert_true(response.bodyUsed, "bodyUsed is true after being consumed"); + }); + }, "Consume empty " + bodyType + " response body as " + (asText ? "text" : "arrayBuffer")); +} + +checkResponseWithEmptyBody("blob", new Blob([], { "type" : "text/plain" }), false); +checkResponseWithEmptyBody("text", "", false); +checkResponseWithEmptyBody("blob", new Blob([], { "type" : "text/plain" }), true); +checkResponseWithEmptyBody("text", "", true); +checkResponseWithEmptyBody("URLSearchParams", new URLSearchParams(""), true); +checkResponseWithEmptyBody("FormData", new FormData(), true); +checkResponseWithEmptyBody("ArrayBuffer", new ArrayBuffer(), true); diff --git a/pkg/download/engine/testdata/wpt/fetch/api/response/response-consume-stream.any.js b/pkg/download/engine/testdata/wpt/fetch/api/response/response-consume-stream.any.js new file mode 100644 index 00000000..f89d7341 --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/response/response-consume-stream.any.js @@ -0,0 +1,80 @@ +// META: global=window,worker +// META: title=Response consume +// META: script=../resources/utils.js + +promise_test(function(test) { + var body = ""; + var response = new Response(""); + return validateStreamFromString(response.body.getReader(), ""); +}, "Read empty text response's body as readableStream"); + +promise_test(function(test) { + var response = new Response(new Blob([], { "type" : "text/plain" })); + return validateStreamFromString(response.body.getReader(), ""); +}, "Read empty blob response's body as readableStream"); + +var formData = new FormData(); +formData.append("name", "value"); +var textData = JSON.stringify("This is response's body"); +var blob = new Blob([textData], { "type" : "text/plain" }); +var urlSearchParamsData = "name=value"; +var urlSearchParams = new URLSearchParams(urlSearchParamsData); + +for (const mode of [undefined, "byob"]) { + promise_test(function(test) { + var response = new Response(blob); + return validateStreamFromString(response.body.getReader({ mode }), textData); + }, `Read blob response's body as readableStream with mode=${mode}`); + + promise_test(function(test) { + var response = new Response(textData); + return validateStreamFromString(response.body.getReader({ mode }), textData); + }, `Read text response's body as readableStream with mode=${mode}`); + + promise_test(function(test) { + var response = new Response(urlSearchParams); + return validateStreamFromString(response.body.getReader({ mode }), urlSearchParamsData); + }, `Read URLSearchParams response's body as readableStream with mode=${mode}`); + + promise_test(function(test) { + var arrayBuffer = new ArrayBuffer(textData.length); + var int8Array = new Int8Array(arrayBuffer); + for (var cptr = 0; cptr < textData.length; cptr++) + int8Array[cptr] = textData.charCodeAt(cptr); + + return validateStreamFromString(new Response(arrayBuffer).body.getReader({ mode }), textData); + }, `Read array buffer response's body as readableStream with mode=${mode}`); + + promise_test(function(test) { + var response = new Response(formData); + return validateStreamFromPartialString(response.body.getReader({ mode }), + "Content-Disposition: form-data; name=\"name\"\r\n\r\nvalue"); + }, `Read form data response's body as readableStream with mode=${mode}`); +} + +test(function() { + assert_equals(Response.error().body, null); +}, "Getting an error Response stream"); + +test(function() { + assert_equals(Response.redirect("/").body, null); +}, "Getting a redirect Response stream"); + +promise_test(async function(test) { + var buffer = new ArrayBuffer(textData.length); + + var body = new Response(textData).body; + const reader = body.getReader( {mode: 'byob'} ); + + let offset = 3; + while (offset < textData.length) { + const {done, value} = await reader.read(new Uint8Array(buffer, offset)); + if (done) { + break; + } + buffer = value.buffer; + offset += value.byteLength; + } + + validateBufferFromString(buffer, `\0\0\0\"This is response's bo`, 'Buffer should be validated'); +}, `Reading with offset from Response stream`); diff --git a/pkg/download/engine/testdata/wpt/fetch/api/response/response-error.any.js b/pkg/download/engine/testdata/wpt/fetch/api/response/response-error.any.js new file mode 100644 index 00000000..a76bc438 --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/response/response-error.any.js @@ -0,0 +1,27 @@ +// META: global=window,worker +// META: title=Response error + +var invalidStatus = [0, 100, 199, 600, 1000]; +invalidStatus.forEach(function(status) { + test(function() { + assert_throws_js(RangeError, function() { new Response("", { "status" : status }); }, + "Expect RangeError exception when status is " + status); + },"Throws RangeError when responseInit's status is " + status); +}); + +var invalidStatusText = ["\n", "Ā"]; +invalidStatusText.forEach(function(statusText) { + test(function() { + assert_throws_js(TypeError, function() { new Response("", { "statusText" : statusText }); }, + "Expect TypeError exception " + statusText); + },"Throws TypeError when responseInit's statusText is " + statusText); +}); + +var nullBodyStatus = [204, 205, 304]; +nullBodyStatus.forEach(function(status) { + test(function() { + assert_throws_js(TypeError, + function() { new Response("body", {"status" : status }); }, + "Expect TypeError exception "); + },"Throws TypeError when building a response with body and a body status of " + status); +}); diff --git a/pkg/download/engine/testdata/wpt/fetch/api/response/response-from-stream.any.js b/pkg/download/engine/testdata/wpt/fetch/api/response/response-from-stream.any.js new file mode 100644 index 00000000..ea5192bf --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/response/response-from-stream.any.js @@ -0,0 +1,23 @@ +// META: global=window,worker + +"use strict"; + +test(() => { + const stream = new ReadableStream(); + stream.getReader(); + assert_throws_js(TypeError, () => new Response(stream)); +}, "Constructing a Response with a stream on which getReader() is called"); + +test(() => { + const stream = new ReadableStream(); + stream.getReader().read(); + assert_throws_js(TypeError, () => new Response(stream)); +}, "Constructing a Response with a stream on which read() is called"); + +promise_test(async () => { + const stream = new ReadableStream({ pull: c => c.enqueue(new Uint8Array()) }), + reader = stream.getReader(); + await reader.read(); + reader.releaseLock(); + assert_throws_js(TypeError, () => new Response(stream)); +}, "Constructing a Response with a stream on which read() and releaseLock() are called"); diff --git a/pkg/download/engine/testdata/wpt/fetch/api/response/response-headers-guard.any.js b/pkg/download/engine/testdata/wpt/fetch/api/response/response-headers-guard.any.js new file mode 100644 index 00000000..4a67d067 --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/response/response-headers-guard.any.js @@ -0,0 +1,8 @@ +// META: global=window,worker +// META: title=Response: error static method + +promise_test (async () => { + const response = await fetch("../resources/data.json"); + assert_throws_js(TypeError, () => { response.headers.append("name", "value"); }); + assert_not_equals(response.headers.get("name"), "value", "response headers should be immutable"); +}, "Ensure response headers are immutable"); diff --git a/pkg/download/engine/testdata/wpt/fetch/api/response/response-init-001.any.js b/pkg/download/engine/testdata/wpt/fetch/api/response/response-init-001.any.js new file mode 100644 index 00000000..559e49ad --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/response/response-init-001.any.js @@ -0,0 +1,64 @@ +// META: global=window,worker +// META: title=Response init: simple cases + +var defaultValues = { "type" : "default", + "url" : "", + "ok" : true, + "status" : 200, + "statusText" : "", + "body" : null +}; + +var statusCodes = { "givenValues" : [200, 300, 400, 500, 599], + "expectedValues" : [200, 300, 400, 500, 599] +}; +var statusTexts = { "givenValues" : ["", "OK", "with space", String.fromCharCode(0x80)], + "expectedValues" : ["", "OK", "with space", String.fromCharCode(0x80)] +}; +var initValuesDict = { "status" : statusCodes, + "statusText" : statusTexts +}; + +function isOkStatus(status) { + return 200 <= status && 299 >= status; +} + +var response = new Response(); +for (var attributeName in defaultValues) { + test(function() { + var expectedValue = defaultValues[attributeName]; + assert_equals(response[attributeName], expectedValue, + "Expect default response." + attributeName + " is " + expectedValue); + }, "Check default value for " + attributeName + " attribute"); +} + +for (var attributeName in initValuesDict) { + test(function() { + var valuesToTest = initValuesDict[attributeName]; + for (var valueIdx in valuesToTest["givenValues"]) { + var givenValue = valuesToTest["givenValues"][valueIdx]; + var expectedValue = valuesToTest["expectedValues"][valueIdx]; + var responseInit = {}; + responseInit[attributeName] = givenValue; + var response = new Response("", responseInit); + assert_equals(response[attributeName], expectedValue, + "Expect response." + attributeName + " is " + expectedValue + + " when initialized with " + givenValue); + assert_equals(response.ok, isOkStatus(response.status), + "Expect response.ok is " + isOkStatus(response.status)); + } + }, "Check " + attributeName + " init values and associated getter"); +} + +test(function() { + const response1 = new Response(""); + assert_equals(response1.headers, response1.headers); + + const response2 = new Response("", {"headers": {"X-Foo": "bar"}}); + assert_equals(response2.headers, response2.headers); + const headers = response2.headers; + response2.headers.set("X-Foo", "quux"); + assert_equals(headers, response2.headers); + headers.set("X-Other-Header", "baz"); + assert_equals(headers, response2.headers); +}, "Test that Response.headers has the [SameObject] extended attribute"); diff --git a/pkg/download/engine/testdata/wpt/fetch/api/response/response-init-002.any.js b/pkg/download/engine/testdata/wpt/fetch/api/response/response-init-002.any.js new file mode 100644 index 00000000..6c0a46e4 --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/response/response-init-002.any.js @@ -0,0 +1,61 @@ +// META: global=window,worker +// META: title=Response init: body and headers +// META: script=../resources/utils.js + +test(function() { + var headerDict = {"name1": "value1", + "name2": "value2", + "name3": "value3" + }; + var headers = new Headers(headerDict); + var response = new Response("", { "headers" : headers }) + for (var name in headerDict) { + assert_equals(response.headers.get(name), headerDict[name], + "response's headers has " + name + " : " + headerDict[name]); + } +}, "Initialize Response with headers values"); + +function checkResponseInit(body, bodyType, expectedTextBody) { + promise_test(function(test) { + var response = new Response(body); + var resHeaders = response.headers; + var mime = resHeaders.get("Content-Type"); + assert_true(mime && mime.search(bodyType) > -1, "Content-Type header should be \"" + bodyType + "\" "); + return response.text().then(function(bodyAsText) { + //not equals: cannot guess formData exact value + assert_true(bodyAsText.search(expectedTextBody) > -1, "Retrieve and verify response body"); + }); + }, "Initialize Response's body with " + bodyType); +} + +var blob = new Blob(["This is a blob"], {type: "application/octet-binary"}); +var formaData = new FormData(); +formaData.append("name", "value"); +var urlSearchParams = "URLSearchParams are not supported"; +//avoid test timeout if not implemented +if (self.URLSearchParams) + urlSearchParams = new URLSearchParams("name=value"); +var usvString = "This is a USVString" + +checkResponseInit(blob, "application/octet-binary", "This is a blob"); +checkResponseInit(formaData, "multipart/form-data", "name=\"name\"\r\n\r\nvalue"); +checkResponseInit(urlSearchParams, "application/x-www-form-urlencoded;charset=UTF-8", "name=value"); +checkResponseInit(usvString, "text/plain;charset=UTF-8", "This is a USVString"); + +promise_test(function(test) { + var body = "This is response body"; + var response = new Response(body); + return validateStreamFromString(response.body.getReader(), body); +}, "Read Response's body as readableStream"); + +promise_test(function(test) { + var response = new Response("This is my fork", {"headers" : [["Content-Type", ""]]}); + return response.blob().then(function(blob) { + assert_equals(blob.type, "", "Blob type should be the empty string"); + }); +}, "Testing empty Response Content-Type header"); + +test(function() { + var response = new Response(null, {status: 204}); + assert_equals(response.body, null); +}, "Testing null Response body"); diff --git a/pkg/download/engine/testdata/wpt/fetch/api/response/response-init-contenttype.any.js b/pkg/download/engine/testdata/wpt/fetch/api/response/response-init-contenttype.any.js new file mode 100644 index 00000000..3a7744c2 --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/response/response-init-contenttype.any.js @@ -0,0 +1,125 @@ +test(() => { + const response = new Response(); + assert_equals(response.headers.get("Content-Type"), null); +}, "Default Content-Type for Response with empty body"); + +test(() => { + const blob = new Blob([]); + const response = new Response(blob); + assert_equals(response.headers.get("Content-Type"), null); +}, "Default Content-Type for Response with Blob body (no type set)"); + +test(() => { + const blob = new Blob([], { type: "" }); + const response = new Response(blob); + assert_equals(response.headers.get("Content-Type"), null); +}, "Default Content-Type for Response with Blob body (empty type)"); + +test(() => { + const blob = new Blob([], { type: "a/b; c=d" }); + const response = new Response(blob); + assert_equals(response.headers.get("Content-Type"), "a/b; c=d"); +}, "Default Content-Type for Response with Blob body (set type)"); + +test(() => { + const buffer = new Uint8Array(); + const response = new Response(buffer); + assert_equals(response.headers.get("Content-Type"), null); +}, "Default Content-Type for Response with buffer source body"); + +promise_test(async () => { + const formData = new FormData(); + formData.append("a", "b"); + const response = new Response(formData); + const boundary = (await response.text()).split("\r\n")[0].slice(2); + assert_equals( + response.headers.get("Content-Type"), + `multipart/form-data; boundary=${boundary}`, + ); +}, "Default Content-Type for Response with FormData body"); + +test(() => { + const usp = new URLSearchParams(); + const response = new Response(usp); + assert_equals( + response.headers.get("Content-Type"), + "application/x-www-form-urlencoded;charset=UTF-8", + ); +}, "Default Content-Type for Response with URLSearchParams body"); + +test(() => { + const response = new Response(""); + assert_equals( + response.headers.get("Content-Type"), + "text/plain;charset=UTF-8", + ); +}, "Default Content-Type for Response with string body"); + +test(() => { + const stream = new ReadableStream(); + const response = new Response(stream); + assert_equals(response.headers.get("Content-Type"), null); +}, "Default Content-Type for Response with ReadableStream body"); + +// ----------------------------------------------------------------------------- + +const OVERRIDE_MIME = "test/only; mime=type"; + +function responseWithOverrideMime(body) { + return new Response( + body, + { headers: { "Content-Type": OVERRIDE_MIME } }, + ); +} + +test(() => { + const response = responseWithOverrideMime(undefined); + assert_equals(response.headers.get("Content-Type"), OVERRIDE_MIME); +}, "Can override Content-Type for Response with empty body"); + +test(() => { + const blob = new Blob([]); + const response = responseWithOverrideMime(blob); + assert_equals(response.headers.get("Content-Type"), OVERRIDE_MIME); +}, "Can override Content-Type for Response with Blob body (no type set)"); + +test(() => { + const blob = new Blob([], { type: "" }); + const response = responseWithOverrideMime(blob); + assert_equals(response.headers.get("Content-Type"), OVERRIDE_MIME); +}, "Can override Content-Type for Response with Blob body (empty type)"); + +test(() => { + const blob = new Blob([], { type: "a/b; c=d" }); + const response = responseWithOverrideMime(blob); + assert_equals(response.headers.get("Content-Type"), OVERRIDE_MIME); +}, "Can override Content-Type for Response with Blob body (set type)"); + +test(() => { + const buffer = new Uint8Array(); + const response = responseWithOverrideMime(buffer); + assert_equals(response.headers.get("Content-Type"), OVERRIDE_MIME); +}, "Can override Content-Type for Response with buffer source body"); + +test(() => { + const formData = new FormData(); + const response = responseWithOverrideMime(formData); + assert_equals(response.headers.get("Content-Type"), OVERRIDE_MIME); +}, "Can override Content-Type for Response with FormData body"); + +test(() => { + const usp = new URLSearchParams(); + const response = responseWithOverrideMime(usp); + assert_equals(response.headers.get("Content-Type"), OVERRIDE_MIME); +}, "Can override Content-Type for Response with URLSearchParams body"); + +test(() => { + const response = responseWithOverrideMime(""); + assert_equals(response.headers.get("Content-Type"), OVERRIDE_MIME); +}, "Can override Content-Type for Response with string body"); + +test(() => { + const stream = new ReadableStream(); + const response = responseWithOverrideMime(stream); + assert_equals(response.headers.get("Content-Type"), OVERRIDE_MIME); +}, "Can override Content-Type for Response with ReadableStream body"); diff --git a/pkg/download/engine/testdata/wpt/fetch/api/response/response-static-error.any.js b/pkg/download/engine/testdata/wpt/fetch/api/response/response-static-error.any.js new file mode 100644 index 00000000..4097eab3 --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/response/response-static-error.any.js @@ -0,0 +1,22 @@ +// META: global=window,worker +// META: title=Response: error static method + +test(function() { + var responseError = Response.error(); + assert_equals(responseError.type, "error", "Network error response's type is error"); + assert_equals(responseError.status, 0, "Network error response's status is 0"); + assert_equals(responseError.statusText, "", "Network error response's statusText is empty"); + assert_equals(responseError.body, null, "Network error response's body is null"); + + assert_true(responseError.headers.entries().next().done, "Headers should be empty"); +}, "Check response returned by static method error()"); + +test(function() { + const headers = Response.error().headers; + + // Avoid false positives if expected API is not available + assert_true(!!headers); + assert_equals(typeof headers.append, 'function'); + + assert_throws_js(TypeError, function () { headers.append('name', 'value'); }); +}, "the 'guard' of the Headers instance should be immutable"); diff --git a/pkg/download/engine/testdata/wpt/fetch/api/response/response-static-json.any.js b/pkg/download/engine/testdata/wpt/fetch/api/response/response-static-json.any.js new file mode 100644 index 00000000..5ec79e69 --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/response/response-static-json.any.js @@ -0,0 +1,96 @@ +// META: global=window,worker +// META: title=Response: json static method + +const APPLICATION_JSON = "application/json"; +const FOO_BAR = "foo/bar"; + +const INIT_TESTS = [ + [undefined, 200, "", APPLICATION_JSON, {}], + [{ status: 400 }, 400, "", APPLICATION_JSON, {}], + [{ statusText: "foo" }, 200, "foo", APPLICATION_JSON, {}], + [{ headers: {} }, 200, "", APPLICATION_JSON, {}], + [{ headers: { "content-type": FOO_BAR } }, 200, "", FOO_BAR, {}], + [{ headers: { "x-foo": "bar" } }, 200, "", APPLICATION_JSON, { "x-foo": "bar" }], +]; + +for (const [init, expectedStatus, expectedStatusText, expectedContentType, expectedHeaders] of INIT_TESTS) { + promise_test(async function () { + const response = Response.json("hello world", init); + assert_equals(response.type, "default", "Response's type is default"); + assert_equals(response.status, expectedStatus, "Response's status is " + expectedStatus); + assert_equals(response.statusText, expectedStatusText, "Response's statusText is " + JSON.stringify(expectedStatusText)); + assert_equals(response.headers.get("content-type"), expectedContentType, "Response's content-type is " + expectedContentType); + for (const key in expectedHeaders) { + assert_equals(response.headers.get(key), expectedHeaders[key], "Response's header " + key + " is " + JSON.stringify(expectedHeaders[key])); + } + + const data = await response.json(); + assert_equals(data, "hello world", "Response's body is 'hello world'"); + }, `Check response returned by static json() with init ${JSON.stringify(init)}`); +} + +const nullBodyStatus = [204, 205, 304]; +for (const status of nullBodyStatus) { + test(function () { + assert_throws_js( + TypeError, + function () { + Response.json("hello world", { status: status }); + }, + ); + }, `Throws TypeError when calling static json() with a status of ${status}`); +} + +promise_test(async function () { + const response = Response.json({ foo: "bar" }); + const data = await response.json(); + assert_equals(typeof data, "object", "Response's json body is an object"); + assert_equals(data.foo, "bar", "Response's json body is { foo: 'bar' }"); +}, "Check static json() encodes JSON objects correctly"); + +test(function () { + assert_throws_js( + TypeError, + function () { + Response.json(Symbol("foo")); + }, + ); +}, "Check static json() throws when data is not encodable"); + +test(function () { + const a = { b: 1 }; + a.a = a; + assert_throws_js( + TypeError, + function () { + Response.json(a); + }, + ); +}, "Check static json() throws when data is circular"); + +promise_test(async function () { + class CustomError extends Error { + name = "CustomError"; + } + assert_throws_js( + CustomError, + function () { + Response.json({ get foo() { throw new CustomError("bar") }}); + } + ) +}, "Check static json() propagates JSON serializer errors"); + +const encodingChecks = [ + ["𝌆", [34, 240, 157, 140, 134, 34]], + ["\uDF06\uD834", [34, 92, 117, 100, 102, 48, 54, 92, 117, 100, 56, 51, 52, 34]], + ["\uDEAD", [34, 92, 117, 100, 101, 97, 100, 34]], +]; + +for (const [input, expected] of encodingChecks) { + promise_test(async function () { + const response = Response.json(input); + const buffer = await response.arrayBuffer(); + const data = new Uint8Array(buffer); + assert_array_equals(data, expected); + }, `Check response returned by static json() with input ${input}`); +} diff --git a/pkg/download/engine/testdata/wpt/fetch/api/response/response-static-redirect.any.js b/pkg/download/engine/testdata/wpt/fetch/api/response/response-static-redirect.any.js new file mode 100644 index 00000000..b16c56d8 --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/response/response-static-redirect.any.js @@ -0,0 +1,40 @@ +// META: global=window,worker +// META: title=Response: redirect static method + +var url = "http://test.url:1234/"; +test(function() { + const redirectResponse = Response.redirect(url); + assert_equals(redirectResponse.type, "default"); + assert_false(redirectResponse.redirected); + assert_false(redirectResponse.ok); + assert_equals(redirectResponse.status, 302, "Default redirect status is 302"); + assert_equals(redirectResponse.headers.get("Location"), url, + "redirected response has Location header with the correct url"); + assert_equals(redirectResponse.statusText, ""); +}, "Check default redirect response"); + +[301, 302, 303, 307, 308].forEach(function(status) { + test(function() { + const redirectResponse = Response.redirect(url, status); + assert_equals(redirectResponse.type, "default"); + assert_false(redirectResponse.redirected); + assert_false(redirectResponse.ok); + assert_equals(redirectResponse.status, status, "Redirect status is " + status); + assert_equals(redirectResponse.headers.get("Location"), url); + assert_equals(redirectResponse.statusText, ""); + }, "Check response returned by static method redirect(), status = " + status); +}); + +test(function() { + var invalidUrl = "http://:This is not an url"; + assert_throws_js(TypeError, function() { Response.redirect(invalidUrl); }, + "Expect TypeError exception"); +}, "Check error returned when giving invalid url to redirect()"); + +var invalidRedirectStatus = [200, 309, 400, 500]; +invalidRedirectStatus.forEach(function(invalidStatus) { + test(function() { + assert_throws_js(RangeError, function() { Response.redirect(url, invalidStatus); }, + "Expect RangeError exception"); + }, "Check error returned when giving invalid status to redirect(), status = " + invalidStatus); +}); diff --git a/pkg/download/engine/testdata/wpt/fetch/api/response/response-stream-bad-chunk.any.js b/pkg/download/engine/testdata/wpt/fetch/api/response/response-stream-bad-chunk.any.js new file mode 100644 index 00000000..8e83cd19 --- /dev/null +++ b/pkg/download/engine/testdata/wpt/fetch/api/response/response-stream-bad-chunk.any.js @@ -0,0 +1,25 @@ +// META: global=window,worker +// META: title=Response causes TypeError from bad chunk type + +function runChunkTest(responseReaderMethod, testDescription) { + promise_test(test => { + let stream = new ReadableStream({ + start(controller) { + controller.enqueue("not Uint8Array"); + controller.close(); + } + }); + + return promise_rejects_js(test, TypeError, + new Response(stream)[responseReaderMethod](), + 'TypeError should propagate' + ) + }, testDescription) +} + +runChunkTest('arrayBuffer', 'ReadableStream with non-Uint8Array chunk passed to Response.arrayBuffer() causes TypeError'); +runChunkTest('blob', 'ReadableStream with non-Uint8Array chunk passed to Response.blob() causes TypeError'); +runChunkTest('bytes', 'ReadableStream with non-Uint8Array chunk passed to Response.bytes() causes TypeError'); +runChunkTest('formData', 'ReadableStream with non-Uint8Array chunk passed to Response.formData() causes TypeError'); +runChunkTest('json', 'ReadableStream with non-Uint8Array chunk passed to Response.json() causes TypeError'); +runChunkTest('text', 'ReadableStream with non-Uint8Array chunk passed to Response.text() causes TypeError'); diff --git a/pkg/download/engine/testdata/wpt_harness.js b/pkg/download/engine/testdata/wpt_harness.js new file mode 100644 index 00000000..e09ac6e6 --- /dev/null +++ b/pkg/download/engine/testdata/wpt_harness.js @@ -0,0 +1,113 @@ +(function () { + const failures = []; + const pending = []; + let passed = 0; + + function format(value) { + if (typeof value === "string") return JSON.stringify(value); + try { return JSON.stringify(value); } catch (_) { return String(value); } + } + + function fail(message) { + throw new Error(message || "assertion failed"); + } + + globalThis.self = globalThis; + globalThis.setup = function (callback) { + callback.call({ add_cleanup() {} }); + }; + globalThis.test = function (callback, name) { + const cleanups = []; + try { + callback.call({ add_cleanup(fn) { cleanups.push(fn); } }); + passed++; + } catch (error) { + failures.push((name || "unnamed test") + ": " + (error && error.stack ? error.stack : error)); + } finally { + for (let index = cleanups.length - 1; index >= 0; index--) { + try { cleanups[index](); } catch (_) {} + } + } + }; + globalThis.promise_test = function (callback, name) { + const cleanups = []; + const context = { + add_cleanup(fn) { cleanups.push(fn); }, + step_timeout(fn, delay) { return setTimeout(fn, delay); }, + unreached_func(message) { return function () { assert_unreached(message); }; }, + }; + const promise = Promise.resolve().then(() => callback.call(context, context)).then(() => { + passed++; + }, (error) => { + failures.push((name || "unnamed promise test") + ": " + (error && error.stack ? error.stack : error)); + }).finally(() => { + for (let index = cleanups.length - 1; index >= 0; index--) { + try { cleanups[index](); } catch (_) {} + } + }); + pending.push(promise); + return promise; + }; + globalThis.assert_true = function (value, message) { + if (value !== true) fail((message || "expected true") + "; got " + format(value)); + }; + globalThis.assert_false = function (value, message) { + if (value !== false) fail((message || "expected false") + "; got " + format(value)); + }; + globalThis.assert_equals = function (actual, expected, message) { + if (!Object.is(actual, expected)) { + fail((message || "values differ") + "; expected " + format(expected) + ", got " + format(actual)); + } + }; + globalThis.assert_not_equals = function (actual, expected, message) { + if (Object.is(actual, expected)) { + fail((message || "values should differ") + "; both were " + format(actual)); + } + }; + globalThis.assert_array_equals = function (actual, expected, message) { + if (!actual || !expected || actual.length !== expected.length) { + fail((message || "arrays differ") + "; expected " + format(expected) + ", got " + format(actual)); + } + for (let index = 0; index < expected.length; index++) { + if (!Object.is(actual[index], expected[index])) { + fail((message || "arrays differ") + " at " + index + "; expected " + format(expected) + ", got " + format(actual)); + } + } + }; + globalThis.assert_nested_array_equals = function (actual, expected, message) { + if (!actual || !expected || actual.length !== expected.length) { + fail((message || "nested arrays differ") + "; expected " + format(expected) + ", got " + format(actual)); + } + for (let index = 0; index < expected.length; index++) { + assert_array_equals(actual[index], expected[index], (message || "nested arrays differ") + " at " + index); + } + }; + globalThis.assert_unreached = function (message) { + fail(message || "assert_unreached was reached"); + }; + globalThis.assert_throws_js = function (constructor, callback, message) { + let thrown = null; + try { callback(); } catch (error) { thrown = error; } + if (thrown == null) fail((message || "expected exception") + "; nothing was thrown"); + if (!(thrown instanceof constructor)) { + fail((message || "wrong exception") + "; expected " + constructor.name + ", got " + thrown); + } + }; + globalThis.promise_rejects_js = function (_test, constructor, promise, message) { + return Promise.resolve(promise).then(() => { + fail((message || "expected promise rejection") + "; promise resolved"); + }, (error) => { + if (!(error instanceof constructor)) { + fail((message || "wrong rejection") + "; expected " + constructor.name + ", got " + error); + } + }); + }; + globalThis.assert_greater_than = function (actual, expected, message) { + if (!(actual > expected)) fail((message || "expected greater value") + "; expected > " + expected + ", got " + actual); + }; + globalThis.__wptFinish = async function () { + await Promise.all(pending); + if (failures.length) throw new Error(failures.join("\n\n")); + return JSON.stringify({ passed, failed: 0 }); + }; +})(); diff --git a/pkg/download/engine/wpt_fetch_test.go b/pkg/download/engine/wpt_fetch_test.go new file mode 100644 index 00000000..86111d10 --- /dev/null +++ b/pkg/download/engine/wpt_fetch_test.go @@ -0,0 +1,120 @@ +package engine_test + +import ( + "embed" + "fmt" + "net/http" + "net/http/httptest" + "path" + "testing" + + "github.com/GopeedLab/gopeed/pkg/download/engine" +) + +//go:embed testdata/wpt_harness.js +var wptHarness string + +//go:embed testdata/wpt/fetch/api/resources/utils.js +var wptFetchUtils string + +//go:embed testdata/wpt/fetch/api/request/request-error.js +var wptRequestError string + +//go:embed testdata/wpt/fetch/api/cors/resources/not-cors-safelisted.json +var wptNoCORSSafelistedHeaders []byte + +//go:embed testdata/wpt/fetch/api/headers/*.any.js +var wptHeaders embed.FS + +//go:embed testdata/wpt/fetch/api/request/*.any.js testdata/wpt/fetch/api/response/*.any.js +var wptRequestResponse embed.FS + +func TestWPTHeaders(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, _ *http.Request) { + response.Header().Set("Content-Type", "application/json") + _, _ = response.Write(wptNoCORSSafelistedHeaders) + })) + t.Cleanup(server.Close) + + tests := []string{ + "header-setcookie.any.js", + "headers-basic.any.js", + "headers-casing.any.js", + "headers-combine.any.js", + "headers-errors.any.js", + "headers-forbidden-override.any.js", + "headers-normalize.any.js", + "headers-no-cors.any.js", + "headers-record.any.js", + "headers-structure.any.js", + } + for _, name := range tests { + t.Run(name, func(t *testing.T) { + setup := "" + if name == "headers-no-cors.any.js" { + setup = fmt.Sprintf("globalThis.location = new URL(%q);", server.URL+"/fetch/api/headers/test.any.js") + } + runWPTFile(t, wptHeaders, path.Join("testdata/wpt/fetch/api/headers", name), setup) + }) + } +} + +func TestWPTRequestResponse(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, _ *http.Request) { + response.Header().Set("Content-Type", "application/json") + _, _ = response.Write([]byte("{}")) + })) + t.Cleanup(server.Close) + setup := fmt.Sprintf("globalThis.location = new URL(%q);", server.URL+"/fetch/api/response/test.any.js") + + tests := []string{ + "request/forbidden-method.any.js", + "request/request-constructor-init-body-override.any.js", + "request/request-consume.any.js", + "request/request-consume-empty.any.js", + "request/request-disturbed.any.js", + "request/request-error.any.js", + "request/request-headers.any.js", + "request/request-init-002.any.js", + "request/request-init-contenttype.any.js", + "request/request-init-priority.any.js", + "request/request-init-stream.any.js", + "request/request-keepalive.any.js", + "request/request-clone-readable-stream-body.any.js", + "request/request-structure.any.js", + "response/response-consume-empty.any.js", + "response/response-consume-stream.any.js", + "response/response-init-001.any.js", + "response/response-init-002.any.js", + "response/response-init-contenttype.any.js", + "response/response-from-stream.any.js", + "response/response-headers-guard.any.js", + "response/response-error.any.js", + "response/response-stream-bad-chunk.any.js", + "response/response-static-error.any.js", + "response/response-static-json.any.js", + "response/response-static-redirect.any.js", + } + for _, name := range tests { + t.Run(name, func(t *testing.T) { + runWPTFile(t, wptRequestResponse, path.Join("testdata/wpt/fetch/api", name), setup) + }) + } +} + +func runWPTFile(t *testing.T, files embed.FS, name, setup string) { + t.Helper() + source, err := files.ReadFile(name) + if err != nil { + t.Fatal(err) + } + runtime := engine.NewEngine(nil) + t.Cleanup(runtime.Close) + value, err := runtime.RunString(setup + "\n" + wptHarness + "\n" + wptFetchUtils + "\n" + wptRequestError + "\n" + string(source) + "\n__wptFinish();") + if err != nil { + t.Fatal(err) + } + if value == nil { + t.Fatal("WPT harness returned no result") + } +} diff --git a/pkg/download/extension_engine_session.go b/pkg/download/extension_engine_session.go index 44b7fa7d..451a2764 100644 --- a/pkg/download/extension_engine_session.go +++ b/pkg/download/extension_engine_session.go @@ -118,8 +118,6 @@ func (d *Downloader) newExtensionEngine() (*engine.Engine, *engineSession) { RevokeObjectURL: func(url string) error { return d.blob.Revoke(url) }, - ProxyHandler: d.cfg.Proxy.ToHandler(), - RegisterCleanup: session.OnClose, } e := engine.NewEngine(&engine.Config{ ProxyConfig: d.cfg.Proxy,