From 8de838316509ea3e83d2144cc07a7f4ab252046e Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Tue, 17 Sep 2024 09:23:30 +0200 Subject: [PATCH 1/4] feat(fetch): expose .timing() make responseEnd optional make it more similar to existing types missed one! --- docs/src/api/class-apiresponse.md | 23 +++++++ docs/src/api/class-request.md | 2 +- packages/playwright-core/src/client/fetch.ts | 7 ++ .../playwright-core/src/protocol/validator.ts | 2 + .../server/dispatchers/networkDispatchers.ts | 4 +- packages/playwright-core/src/server/fetch.ts | 18 +++++- packages/playwright-core/types/types.d.ts | 64 ++++++++++++++++++- packages/protocol/src/channels.ts | 2 + packages/protocol/src/protocol.yml | 2 + tests/library/browsercontext-fetch.spec.ts | 11 ++++ 10 files changed, 130 insertions(+), 5 deletions(-) diff --git a/docs/src/api/class-apiresponse.md b/docs/src/api/class-apiresponse.md index 5a901b76ba949..5f57a3b3912a8 100644 --- a/docs/src/api/class-apiresponse.md +++ b/docs/src/api/class-apiresponse.md @@ -110,3 +110,26 @@ Returns the text representation of response body. - returns: <[string]> Contains the URL of the response. + +## method: APIResponse.timing +* since: v1.48 +- returns: <[Object]> + - `startTime` <[float]> Request start time in milliseconds elapsed since January 1, 1970 00:00:00 UTC + - `domainLookupStart` <[float]> Time immediately before the client starts the domain name lookup for the + resource. The value is given in milliseconds relative to `startTime`, -1 if not available. + - `domainLookupEnd` <[float]> Time immediately after the client ends the domain name lookup for the resource. + The value is given in milliseconds relative to `startTime`, -1 if not available. + - `connectStart` <[float]> Time immediately before the client starts establishing the connection to the server + to retrieve the resource. The value is given in milliseconds relative to `startTime`, -1 if not available. + - `secureConnectionStart` <[float]> Time immediately before the client starts the handshake process to secure the + current connection. The value is given in milliseconds relative to `startTime`, -1 if not available. + - `connectEnd` <[float]> Time immediately before the client starts establishing the connection to the server + to retrieve the resource. The value is given in milliseconds relative to `startTime`, -1 if not available. + - `requestStart` <[float]> Time immediately before the client starts requesting the resource from the server. The value is given in milliseconds relative to `startTime`, -1 if not available. + - `responseStart` <[float]> Time immediately after the client receives the first byte of the response from the server. The value is given in milliseconds relative to `startTime`, -1 if not available. + - `responseEnd` <[float]> Time immediately after the client receives the last byte of the resource or immediately + before the transport connection is closed, whichever comes first. The value is given in milliseconds relative to + `startTime`, -1 if not available. + +Returns resource timing information for given response. Find more information at +[Resource Timing API](https://developer.mozilla.org/en-US/docs/Web/API/PerformanceResourceTiming). diff --git a/docs/src/api/class-request.md b/docs/src/api/class-request.md index e13d9de69f32f..a9f2d307e5b08 100644 --- a/docs/src/api/class-request.md +++ b/docs/src/api/class-request.md @@ -314,7 +314,7 @@ Returns resource size information for given request. - `startTime` <[float]> Request start time in milliseconds elapsed since January 1, 1970 00:00:00 UTC - `domainLookupStart` <[float]> Time immediately before the browser starts the domain name lookup for the resource. The value is given in milliseconds relative to `startTime`, -1 if not available. - - `domainLookupEnd` <[float]> Time immediately after the browser starts the domain name lookup for the resource. + - `domainLookupEnd` <[float]> Time immediately after the browser ends the domain name lookup for the resource. The value is given in milliseconds relative to `startTime`, -1 if not available. - `connectStart` <[float]> Time immediately before the user agent starts establishing the connection to the server to retrieve the resource. The value is given in milliseconds relative to `startTime`, -1 if not available. diff --git a/packages/playwright-core/src/client/fetch.ts b/packages/playwright-core/src/client/fetch.ts index 58928532ac9e4..a86a87f362c36 100644 --- a/packages/playwright-core/src/client/fetch.ts +++ b/packages/playwright-core/src/client/fetch.ts @@ -332,6 +332,13 @@ export class APIResponse implements api.APIResponse { return this._headers.headersArray(); } + timing() { + return { + ...this._initializer.timing, + responseEnd: this._initializer.responseEndTiming, + }; + } + async body(): Promise { try { const result = await this._request._channel.fetchResponseBody({ fetchUid: this._fetchUid() }); diff --git a/packages/playwright-core/src/protocol/validator.ts b/packages/playwright-core/src/protocol/validator.ts index abea7f8fce57c..369568a8086e6 100644 --- a/packages/playwright-core/src/protocol/validator.ts +++ b/packages/playwright-core/src/protocol/validator.ts @@ -224,6 +224,8 @@ scheme.APIResponse = tObject({ status: tNumber, statusText: tString, headers: tArray(tType('NameValue')), + timing: tType('ResourceTiming'), + responseEndTiming: tNumber, }); scheme.LifecycleEvent = tEnum(['load', 'domcontentloaded', 'networkidle', 'commit']); scheme.LocalUtilsInitializer = tObject({ diff --git a/packages/playwright-core/src/server/dispatchers/networkDispatchers.ts b/packages/playwright-core/src/server/dispatchers/networkDispatchers.ts index ba600f697e7d7..e387ff1cbd9e9 100644 --- a/packages/playwright-core/src/server/dispatchers/networkDispatchers.ts +++ b/packages/playwright-core/src/server/dispatchers/networkDispatchers.ts @@ -212,7 +212,9 @@ export class APIRequestContextDispatcher extends Dispatcher { + const endAt = monotonicTime(); + // spec: https://developer.mozilla.org/en-US/docs/Web/API/PerformanceResourceTiming + const timing: channels.ResourceTiming = { + startTime: startAt, + domainLookupStart: dnsLookupAt ? 0 : -1, + domainLookupEnd: dnsLookupAt ? dnsLookupAt! - startAt : -1, + connectStart: dnsLookupAt ? dnsLookupAt! - startAt : 0, + secureConnectionStart: dnsLookupAt ? dnsLookupAt! - startAt : 0, + connectEnd: (tlsHandshakeAt ?? tcpConnectionAt!) - startAt, + requestStart: (tlsHandshakeAt ?? tcpConnectionAt!) - startAt, + responseStart: responseAt - startAt, + }; + const responseEndTiming = endAt - startAt; + const body = Buffer.concat(chunks); notifyRequestFinished(body); fulfill({ @@ -429,7 +443,9 @@ export abstract class APIRequestContext extends SdkObject { status: response.statusCode || 0, statusText: response.statusMessage || '', headers: toHeadersArray(response.rawHeaders), - body + body, + timing, + responseEndTiming, }); }; diff --git a/packages/playwright-core/types/types.d.ts b/packages/playwright-core/types/types.d.ts index ad102f9271f72..008b0d426d5ef 100644 --- a/packages/playwright-core/types/types.d.ts +++ b/packages/playwright-core/types/types.d.ts @@ -17366,6 +17366,66 @@ export interface APIResponse { */ text(): Promise; + /** + * Returns resource timing information for given response. Find more information at + * [Resource Timing API](https://developer.mozilla.org/en-US/docs/Web/API/PerformanceResourceTiming). + */ + timing(): { + /** + * Request start time in milliseconds elapsed since January 1, 1970 00:00:00 UTC + */ + startTime: number; + + /** + * Time immediately before the client starts the domain name lookup for the resource. The value is given in + * milliseconds relative to `startTime`, -1 if not available. + */ + domainLookupStart: number; + + /** + * Time immediately after the client ends the domain name lookup for the resource. The value is given in milliseconds + * relative to `startTime`, -1 if not available. + */ + domainLookupEnd: number; + + /** + * Time immediately before the client starts establishing the connection to the server to retrieve the resource. The + * value is given in milliseconds relative to `startTime`, -1 if not available. + */ + connectStart: number; + + /** + * Time immediately before the client starts the handshake process to secure the current connection. The value is + * given in milliseconds relative to `startTime`, -1 if not available. + */ + secureConnectionStart: number; + + /** + * Time immediately before the client starts establishing the connection to the server to retrieve the resource. The + * value is given in milliseconds relative to `startTime`, -1 if not available. + */ + connectEnd: number; + + /** + * Time immediately before the client starts requesting the resource from the server. The value is given in + * milliseconds relative to `startTime`, -1 if not available. + */ + requestStart: number; + + /** + * Time immediately after the client receives the first byte of the response from the server. The value is given in + * milliseconds relative to `startTime`, -1 if not available. + */ + responseStart: number; + + /** + * Time immediately after the client receives the last byte of the resource or immediately before the transport + * connection is closed, whichever comes first. The value is given in milliseconds relative to `startTime`, -1 if not + * available. + */ + responseEnd: number; + }; + /** * Contains the URL of the response. */ @@ -19307,8 +19367,8 @@ export interface Request { domainLookupStart: number; /** - * Time immediately after the browser starts the domain name lookup for the resource. The value is given in - * milliseconds relative to `startTime`, -1 if not available. + * Time immediately after the browser ends the domain name lookup for the resource. The value is given in milliseconds + * relative to `startTime`, -1 if not available. */ domainLookupEnd: number; diff --git a/packages/protocol/src/channels.ts b/packages/protocol/src/channels.ts index 689f0275b19f1..6a7ec6e0cff65 100644 --- a/packages/protocol/src/channels.ts +++ b/packages/protocol/src/channels.ts @@ -398,6 +398,8 @@ export type APIResponse = { status: number, statusText: string, headers: NameValue[], + timing: ResourceTiming, + responseEndTiming: number, }; export type LifecycleEvent = 'load' | 'domcontentloaded' | 'networkidle' | 'commit'; diff --git a/packages/protocol/src/protocol.yml b/packages/protocol/src/protocol.yml index ce206ab569a67..1a6d4801ca6cb 100644 --- a/packages/protocol/src/protocol.yml +++ b/packages/protocol/src/protocol.yml @@ -353,6 +353,8 @@ APIResponse: headers: type: array items: NameValue + timing: ResourceTiming + responseEndTiming: number LifecycleEvent: diff --git a/tests/library/browsercontext-fetch.spec.ts b/tests/library/browsercontext-fetch.spec.ts index d10b182cf036b..eb24bd939ff31 100644 --- a/tests/library/browsercontext-fetch.spec.ts +++ b/tests/library/browsercontext-fetch.spec.ts @@ -52,6 +52,17 @@ it('fetch should work', async ({ context, server }) => { expect(response.ok()).toBeTruthy(); expect(response.headers()['content-type']).toBe('application/json; charset=utf-8'); expect(response.headersArray()).toContainEqual({ name: 'Content-Type', value: 'application/json; charset=utf-8' }); + expect(response.timing()).toEqual({ + connectEnd: expect.any(Number), + connectStart: expect.any(Number), + domainLookupEnd: expect.any(Number), + domainLookupStart: expect.any(Number), + requestStart: expect.any(Number), + responseStart: expect.any(Number), + responseEnd: expect.any(Number), + secureConnectionStart: expect.any(Number), + startTime: expect.any(Number), + }); expect(await response.text()).toBe('{"foo": "bar"}\n'); }); From 819e0a5275ddc6738a39e5dc8a0ffda5694be2bd Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Tue, 17 Sep 2024 19:29:37 +0200 Subject: [PATCH 2/4] fix bug with socks/http proxy --- packages/playwright-core/src/server/fetch.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/playwright-core/src/server/fetch.ts b/packages/playwright-core/src/server/fetch.ts index 5e95938e8b163..39df932a50e4b 100644 --- a/packages/playwright-core/src/server/fetch.ts +++ b/packages/playwright-core/src/server/fetch.ts @@ -422,6 +422,11 @@ export abstract class APIRequestContext extends SdkObject { const chunks: Buffer[] = []; const notifyBodyFinished = () => { + function relativeTime(time: number | undefined): number { + if (!time) + return -1; + return time - startAt; + } const endAt = monotonicTime(); // spec: https://developer.mozilla.org/en-US/docs/Web/API/PerformanceResourceTiming const timing: channels.ResourceTiming = { @@ -430,8 +435,8 @@ export abstract class APIRequestContext extends SdkObject { domainLookupEnd: dnsLookupAt ? dnsLookupAt! - startAt : -1, connectStart: dnsLookupAt ? dnsLookupAt! - startAt : 0, secureConnectionStart: dnsLookupAt ? dnsLookupAt! - startAt : 0, - connectEnd: (tlsHandshakeAt ?? tcpConnectionAt!) - startAt, - requestStart: (tlsHandshakeAt ?? tcpConnectionAt!) - startAt, + connectEnd: relativeTime(tlsHandshakeAt ?? tcpConnectionAt), + requestStart: relativeTime(tlsHandshakeAt ?? tcpConnectionAt), responseStart: responseAt - startAt, }; const responseEndTiming = endAt - startAt; @@ -499,6 +504,9 @@ export abstract class APIRequestContext extends SdkObject { socket.on('lookup', () => { dnsLookupAt = monotonicTime(); }); socket.on('connect', () => { tcpConnectionAt = monotonicTime(); }); socket.on('secureConnect', () => { tlsHandshakeAt = monotonicTime(); }); + + // socks / http proxy + socket.on('proxyConnect', () => { tcpConnectionAt = monotonicTime(); }); }); request.on('finish', () => { requestFinishAt = monotonicTime(); }); From b717257013f459955f0172483e1b52b55ac7305d Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Wed, 18 Sep 2024 15:51:19 +0200 Subject: [PATCH 3/4] startAt needs to be walltime --- packages/playwright-core/src/server/fetch.ts | 3 ++- tests/library/browsercontext-fetch.spec.ts | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/playwright-core/src/server/fetch.ts b/packages/playwright-core/src/server/fetch.ts index c1e394e24972d..dbbcb8f80e772 100644 --- a/packages/playwright-core/src/server/fetch.ts +++ b/packages/playwright-core/src/server/fetch.ts @@ -302,6 +302,7 @@ export abstract class APIRequestContext extends SdkObject { const requestOptions = { ...options, agent }; const startAt = monotonicTime(); + const startAtWallTime = Date.now(); let dnsLookupAt: number | undefined; let tcpConnectionAt: number | undefined; let tlsHandshakeAt: number | undefined; @@ -441,7 +442,7 @@ export abstract class APIRequestContext extends SdkObject { const endAt = monotonicTime(); // spec: https://developer.mozilla.org/en-US/docs/Web/API/PerformanceResourceTiming const timing: channels.ResourceTiming = { - startTime: startAt, + startTime: startAtWallTime, domainLookupStart: dnsLookupAt ? 0 : -1, domainLookupEnd: dnsLookupAt ? dnsLookupAt! - startAt : -1, connectStart: dnsLookupAt ? dnsLookupAt! - startAt : 0, diff --git a/tests/library/browsercontext-fetch.spec.ts b/tests/library/browsercontext-fetch.spec.ts index eb24bd939ff31..69cff23fe7528 100644 --- a/tests/library/browsercontext-fetch.spec.ts +++ b/tests/library/browsercontext-fetch.spec.ts @@ -63,6 +63,7 @@ it('fetch should work', async ({ context, server }) => { secureConnectionStart: expect.any(Number), startTime: expect.any(Number), }); + expect(response.timing().startTime, 'is right order of magnitude').toBeCloseTo(Date.now(), -3); expect(await response.text()).toBe('{"foo": "bar"}\n'); }); From b1d523b009dc99d856a9dd870dfd639f4671fd00 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Wed, 18 Sep 2024 16:11:37 +0200 Subject: [PATCH 4/4] some more relativeTime --- packages/playwright-core/src/server/fetch.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/playwright-core/src/server/fetch.ts b/packages/playwright-core/src/server/fetch.ts index dbbcb8f80e772..0e2dac44e2c9b 100644 --- a/packages/playwright-core/src/server/fetch.ts +++ b/packages/playwright-core/src/server/fetch.ts @@ -444,12 +444,12 @@ export abstract class APIRequestContext extends SdkObject { const timing: channels.ResourceTiming = { startTime: startAtWallTime, domainLookupStart: dnsLookupAt ? 0 : -1, - domainLookupEnd: dnsLookupAt ? dnsLookupAt! - startAt : -1, - connectStart: dnsLookupAt ? dnsLookupAt! - startAt : 0, - secureConnectionStart: dnsLookupAt ? dnsLookupAt! - startAt : 0, + domainLookupEnd: relativeTime(dnsLookupAt), + connectStart: dnsLookupAt ? relativeTime(dnsLookupAt) : 0, + secureConnectionStart: tlsHandshakeAt ? relativeTime(dnsLookupAt) : 0, connectEnd: relativeTime(tlsHandshakeAt ?? tcpConnectionAt), requestStart: relativeTime(tlsHandshakeAt ?? tcpConnectionAt), - responseStart: responseAt - startAt, + responseStart: relativeTime(responseAt), }; const responseEndTiming = endAt - startAt;