') + ')', 'gi'); if (regex.test(text)) { found = true; var frag = document.createDocumentFragment(); var parts = text.split(regex); parts.forEach(function(part, i) { if (i % 2 === 0) { frag.appendChild(document.createTextNode(part)); } else { var span = document.createElement('span'); span.className = 'userscript-highlight'; span.textContent = part; frag.appendChild(span); } }); node.parentNode.replaceChild(frag, node); } }); } else if (node.nodeType === 1 && node.childNodes) { // element var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT']; if (!skipTags.includes(node.tagName)) { Array.from(node.childNodes).forEach(highlight); } } } highlight(document.body); // Re-highlight on dynamic content var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1 || node.nodeType === 3) highlight(node); }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Strip utm_, fbclid, gclid, etc. from all links on page (function() { var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid', 'ref', 'ref_src', 'source', 'medium', 'campaign']; function cleanUrl(url) { try { var u = new URL(url, window.location.origin); var changed = false; trackingParams.forEach(function(p) { if (u.searchParams.has(p)) { u.searchParams.delete(p); changed = true; } }); return changed ? u.toString() : url; } catch (e) { return url; } } function cleanLinks() { document.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } cleanLinks(); var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1) { if (node.tagName === 'A') cleanLinks(); node.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Auto-enable theater mode on YouTube (function() { function tryTheater() { var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]'); if (btn && !btn.classList.contains('activated')) { btn.click(); } } // Try immediately tryTheater(); // Try after navigation (SPA) var lastUrl = location.href; setInterval(function() { if (location.href !== lastUrl) { lastUrl = location.href; setTimeout(tryTheater, 500); } }, 1000); // Also try on player load var observer = new MutationObserver(tryTheater); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Remove or un-stick sticky/fixed headers that block content (function() { function unstick() { document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) { if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') { el.style.position = 'static'; el.style.top = 'auto'; el.style.zIndex = 'auto'; } }); } unstick(); var observer = new MutationObserver(unstick); observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] }); })(); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); })(); Extend support for `useFileOutput` to `stream` by mattt · Pull Request #309 · replicate/replicate-javascript · GitHub
Skip to content

Extend support for useFileOutput to stream#309

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mattt merged 5 commits into from
Sep 25, 2024
Merged

Extend support for useFileOutput to stream#309

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
254b7df
Break stream on done event without enqueuing it
mattt Sep 23, 2024
4430e6b
Fix doc comment for fetch parameter
mattt Sep 23, 2024
139fb89
Transform URLs in streaming responses when useFileOutput is enabled
mattt Sep 23, 2024
6a2156d
Revert 'Break stream on done event without enqueuing it'
mattt Sep 23, 2024
51d300b
Default to useFileOutput = true for readable streams
mattt Sep 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 15 additions & 6 deletions lib/stream.js
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -48,15 +48,18 @@ class ServerSentEvent {
* @param {string} config.url The URL to connect to.
* @param {typeof fetch} [config.fetch] The URL to connect to.
* @param {object} [config.options] The EventSource options.
* @param {boolean} [config.options.useFileOutput] Whether to use the file output stream.
* @returns {ReadableStream<ServerSentEvent> & AsyncIterable<ServerSentEvent>}
*/
function createReadableStream({ url, fetch, options = {} }) {
const { useFileOutput = true, headers = {}, ...initOptions } = options;

return new ReadableStream({
async start(controller) {
const init = {
...options,
...initOptions,
headers: {
...options.headers,
...headers,
Accept: "text/event-stream",
},
};
Expand DownExpand Up@@ -84,9 +87,15 @@ function createReadableStream({ url, fetch, options = {} }) {
break;
}

controller.enqueue(
new ServerSentEvent(event.event, event.data, event.id)
);
let data = event.data;
if (
useFileOutput &&
typeof data === "string" &&
(data.startsWith("https:") || data.startsWith("data:"))

@aronaronSep 23, 2024

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we want to explicitly match that data consists of only a valid data uri. There is a reasonable chance that a language model might emit a data: line that starts with "data:" but less so that it will emit a line that consists only of a well formed data-uri.

🤔 That said, I did think today after reading this post on other AI apis that we should move to structured outputs.

Perhaps the file stream should emit JSON.

data: {"type": "url", value: "data://..."}

And in future we refactor the text streaming interface to do the same:

data: {"type": "string", "data: and some more text"}

@matttmatttSep 25, 2024

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we want to explicitly match that data consists of only a valid data uri. There is a reasonable chance that a language model might emit a data: line that starts with "data:" but less so that it will emit a line that consists only of a well formed data-uri.

That's a good callout. The trick is finding a good way to validate without parsing the whole thing and throwing away the result. I think we can still read lazily if we use a regex to apply some heuristics about its first chunk of content.

🤔 That said, I did think today after reading this post on other AI apis that we should move to structured outputs.

Perhaps the file stream should emit JSON.

data: {"type": "url", value: "data://..."}

And in future we refactor the text streaming interface to do the same:

data: {"type": "string", "data: and some more text"}

I see the advantages of typed outputs, but also quite like the experience we have now of emitting raw tokens. In any case, structured outputs would be a backwards-incompatible change, so we'd have to be clever about a migration.

One way we could support both would be keeping Accept: text/event-stream as-is, but adding support for Accept: text/event-stream+json. The client libraries could start sending that as needed to opt into structured outputs.

) {
data = createFileOutput({ data, fetch });
}
controller.enqueue(new ServerSentEvent(event.event, data, event.id));

if (event.event === "done") {
break;
Expand All@@ -104,7 +113,7 @@ function createReadableStream({ url, fetch, options = {} }) {
*
* @param {object} config
* @param {string} config.url The URL to connect to.
* @param {typeof fetch} [config.fetch] The URL to connect to.
* @param {typeof fetch} [config.fetch] The fetch function.
* @returns {ReadableStream<Uint8Array>}
*/
function createFileOutput({ url, fetch }) {
Expand Down