From 5d41d83cbf3ee87ba4241a10bbbd6a7fda3e43d8 Mon Sep 17 00:00:00 2001 From: Harlan Wilton Date: Wed, 27 May 2026 18:05:17 +1000 Subject: [PATCH 1/4] fix(instagram-embed): use facebookexternalhit UA for proxy fetch Googlebot is IP-verified by Instagram, so the proxy returned an empty JS shell (just splash-screen + has-finished-comet-page) from hosts outside Google's IP ranges, e.g. Cloudflare/Vercel. facebookexternalhit is Meta's own crawler and is not IP-verified. Fixes #794 --- packages/script/src/runtime/server/instagram-embed.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/script/src/runtime/server/instagram-embed.ts b/packages/script/src/runtime/server/instagram-embed.ts index 4d3173bed..49a921f13 100644 --- a/packages/script/src/runtime/server/instagram-embed.ts +++ b/packages/script/src/runtime/server/instagram-embed.ts @@ -81,7 +81,11 @@ export default withSigning(defineEventHandler(async (event) => { const html = await cachedEmbedFetch(embedUrl, { headers: { 'Accept': 'text/html', - 'User-Agent': 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)', + // Meta's own crawler UA. Googlebot's UA is also accepted by Instagram + // but is IP-verified, so it fails from hosts outside Google's ranges + // (e.g. Cloudflare/Vercel) and Instagram serves the JS shell instead + // of the SSR'd post. + 'User-Agent': 'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)', }, }).catch((error: any) => { throw createError({ From 5ce7b2fbc9b54eff99bb8afd0d54fc453899544d Mon Sep 17 00:00:00 2001 From: Harlan Wilton Date: Wed, 27 May 2026 18:09:34 +1000 Subject: [PATCH 2/4] fix(instagram-embed): don't cache empty embed shells MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instagram serves a JS-only shell (splash-screen + has-finished-comet-page, no post markup) when it can't render server-side — removed/private posts, unverified bot UAs. Previously cached for 10min, hiding the real post even after upstream recovered. Throw inside the cached function on shell detection so nitro skips the write and the next request refetches. --- .../src/runtime/server/instagram-embed.ts | 51 ++++++++++++++----- 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/packages/script/src/runtime/server/instagram-embed.ts b/packages/script/src/runtime/server/instagram-embed.ts index 49a921f13..4aab3302c 100644 --- a/packages/script/src/runtime/server/instagram-embed.ts +++ b/packages/script/src/runtime/server/instagram-embed.ts @@ -1,5 +1,6 @@ import { createError, defineEventHandler, getQuery, setHeader } from 'h3' -import { useRuntimeConfig } from 'nitropack/runtime' +import { defineCachedFunction, useRuntimeConfig } from 'nitropack/runtime' +import { $fetch } from 'ofetch' import { ELEMENT_NODE, parse, renderSync, TEXT_NODE, walkSync } from 'ultrahtml' import { createCachedJsonFetch } from './utils/cached-upstream' import { proxyAssetUrl, rewriteUrl, rewriteUrlsInText, RSRC_RE, scopeCss } from './utils/instagram-embed' @@ -10,12 +11,38 @@ export { proxyAssetUrl, proxyImageUrl, rewriteUrl, rewriteUrlsInText, scopeCss } const EMBED_INSTAGRAM_SUFFIX_RE = /\/embed\/instagram$/ const SRCSET_SPLIT_RE = /\s+/ +// Instagram serves a JS-only shell (splash screen + comet sentinel, no SSR'd +// post markup) when it can't or won't render server-side — e.g. for bot UAs +// it can't verify, or for removed/private posts. Caching that shell would +// hide the real post for the full 10min window even after upstream recovers. +const SHELL_BODY_RE = /id="(?:splash-screen|has-finished-comet-page)"/ +const HAS_POST_CONTENT_RE = /class="(?:Embed|EmbeddedMedia)"/ + +function isEmbedShell(html: string): boolean { + return SHELL_BODY_RE.test(html) && !HAS_POST_CONTENT_RE.test(html) +} + // Instagram embed HTML is semi-fresh (likes, captions may update); 10min // matches the outbound Cache-Control header and dedupes per post+captions. -const cachedEmbedFetch = createCachedJsonFetch( - 'nuxt-scripts-instagram-embed', - 600, - url => url, +// Throws on shell responses so nitro doesn't cache them. +const cachedEmbedFetch = defineCachedFunction( + async (url: string, headers: Record): Promise => { + const html = await $fetch(url, { timeout: 10000, headers }) + if (isEmbedShell(html)) { + throw createError({ + statusCode: 502, + statusMessage: 'Instagram returned an empty embed shell (post unavailable or upstream rate-limiting)', + }) + } + return html + }, + { + name: 'nuxt-scripts-instagram-embed', + maxAge: 600, + swr: true, + staleMaxAge: 600, + getKey: (url: string) => url, + }, ) // Static CSS from Instagram's CDN is versioned; 24h cache is safe because the @@ -79,14 +106,12 @@ export default withSigning(defineEventHandler(async (event) => { const embedUrl = `${cleanUrl}embed/${captions ? 'captioned/' : ''}` const html = await cachedEmbedFetch(embedUrl, { - headers: { - 'Accept': 'text/html', - // Meta's own crawler UA. Googlebot's UA is also accepted by Instagram - // but is IP-verified, so it fails from hosts outside Google's ranges - // (e.g. Cloudflare/Vercel) and Instagram serves the JS shell instead - // of the SSR'd post. - 'User-Agent': 'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)', - }, + 'Accept': 'text/html', + // Meta's own crawler UA. Googlebot's UA is also accepted by Instagram + // but is IP-verified, so it fails from hosts outside Google's ranges + // (e.g. Cloudflare/Vercel) and Instagram serves the JS shell instead + // of the SSR'd post. + 'User-Agent': 'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)', }).catch((error: any) => { throw createError({ statusCode: error.statusCode || 500, From 77eb5c68af11c6de0115c0360f044758cb1cecbc Mon Sep 17 00:00:00 2001 From: Harlan Wilton Date: Wed, 27 May 2026 18:21:11 +1000 Subject: [PATCH 3/4] test(instagram-embed): cover isEmbedShell detection Move the shell-response check into the shared utils module and add unit tests covering: pure shell, real post that also contains the comet sentinel, real post with Embed wrapper, and unrelated HTML. --- .../src/runtime/server/instagram-embed.ts | 13 +---------- .../runtime/server/utils/instagram-embed.ts | 11 +++++++++ test/unit/instagram-embed.test.ts | 23 +++++++++++++++++++ 3 files changed, 35 insertions(+), 12 deletions(-) diff --git a/packages/script/src/runtime/server/instagram-embed.ts b/packages/script/src/runtime/server/instagram-embed.ts index 4aab3302c..9de193c8f 100644 --- a/packages/script/src/runtime/server/instagram-embed.ts +++ b/packages/script/src/runtime/server/instagram-embed.ts @@ -3,7 +3,7 @@ import { defineCachedFunction, useRuntimeConfig } from 'nitropack/runtime' import { $fetch } from 'ofetch' import { ELEMENT_NODE, parse, renderSync, TEXT_NODE, walkSync } from 'ultrahtml' import { createCachedJsonFetch } from './utils/cached-upstream' -import { proxyAssetUrl, rewriteUrl, rewriteUrlsInText, RSRC_RE, scopeCss } from './utils/instagram-embed' +import { isEmbedShell, proxyAssetUrl, rewriteUrl, rewriteUrlsInText, RSRC_RE, scopeCss } from './utils/instagram-embed' import { withSigning } from './utils/withSigning' export { proxyAssetUrl, proxyImageUrl, rewriteUrl, rewriteUrlsInText, scopeCss } from './utils/instagram-embed' @@ -11,17 +11,6 @@ export { proxyAssetUrl, proxyImageUrl, rewriteUrl, rewriteUrlsInText, scopeCss } const EMBED_INSTAGRAM_SUFFIX_RE = /\/embed\/instagram$/ const SRCSET_SPLIT_RE = /\s+/ -// Instagram serves a JS-only shell (splash screen + comet sentinel, no SSR'd -// post markup) when it can't or won't render server-side — e.g. for bot UAs -// it can't verify, or for removed/private posts. Caching that shell would -// hide the real post for the full 10min window even after upstream recovers. -const SHELL_BODY_RE = /id="(?:splash-screen|has-finished-comet-page)"/ -const HAS_POST_CONTENT_RE = /class="(?:Embed|EmbeddedMedia)"/ - -function isEmbedShell(html: string): boolean { - return SHELL_BODY_RE.test(html) && !HAS_POST_CONTENT_RE.test(html) -} - // Instagram embed HTML is semi-fresh (likes, captions may update); 10min // matches the outbound Cache-Control header and dedupes per post+captions. // Throws on shell responses so nitro doesn't cache them. diff --git a/packages/script/src/runtime/server/utils/instagram-embed.ts b/packages/script/src/runtime/server/utils/instagram-embed.ts index 352b8199a..ddf9c8c96 100644 --- a/packages/script/src/runtime/server/utils/instagram-embed.ts +++ b/packages/script/src/runtime/server/utils/instagram-embed.ts @@ -1,6 +1,17 @@ import { buildProxyUrl } from './proxy-url' export const RSRC_RE = /url\(\/rsrc\.php([^)]+)\)/g + +// Instagram serves a JS-only shell (splash-screen + comet sentinel, no SSR'd +// post markup) when it can't or won't render server-side — e.g. for bot UAs +// it can't verify, or for removed/private posts. +const SHELL_BODY_RE = /id="(?:splash-screen|has-finished-comet-page)"/ +const HAS_POST_CONTENT_RE = /class="(?:Embed|EmbeddedMedia)"/ + +export function isEmbedShell(html: string): boolean { + return SHELL_BODY_RE.test(html) && !HAS_POST_CONTENT_RE.test(html) +} + export const AMP_RE = /&/g export const SCONTENT_RE = /https:\/\/scontent[^"'\s),]+\.cdninstagram\.com[^"'\s),]+/g export const STATIC_CDN_RE = /https:\/\/static\.cdninstagram\.com[^"'\s),]+/g diff --git a/test/unit/instagram-embed.test.ts b/test/unit/instagram-embed.test.ts index e57f441c6..029fa1b35 100644 --- a/test/unit/instagram-embed.test.ts +++ b/test/unit/instagram-embed.test.ts @@ -1,6 +1,7 @@ import { ELEMENT_NODE, parse, renderSync, TEXT_NODE, walkSync } from 'ultrahtml' import { describe, expect, it } from 'vitest' import { + isEmbedShell, proxyImageUrl, rewriteUrl, rewriteUrlsInText, @@ -302,3 +303,25 @@ html, body { margin: 0; padding: 0; } expect(result).toContain(`${scope} [data-x="a,b"] .Embed`) }) }) + +describe('instagram-embed: isEmbedShell', () => { + it('detects JS-only shell with splash-screen and no post markup', () => { + const html = '
' + expect(isEmbedShell(html)).toBe(true) + }) + + it('accepts real SSR\'d post even when shell sentinels appear elsewhere', () => { + // Some Instagram responses include the comet sentinel alongside real content. + const html = '
' + expect(isEmbedShell(html)).toBe(false) + }) + + it('accepts real post with Embed wrapper', () => { + const html = '
content
' + expect(isEmbedShell(html)).toBe(false) + }) + + it('returns false on unrelated HTML (no shell sentinels)', () => { + expect(isEmbedShell('nothing here')).toBe(false) + }) +}) From 6175d578c6844e8e43dc20d56ae0ff2ea7fe47f6 Mon Sep 17 00:00:00 2001 From: Harlan Wilton Date: Wed, 27 May 2026 18:41:14 +1000 Subject: [PATCH 4/4] fix(instagram-embed): bust cache key, broaden shell-content detection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Bump cache name to `nuxt-scripts-instagram-embed-v2` to evict any empty-shell entries cached under v1 before the fix. - Include headers in the cache key — Instagram's response is UA-dependent, so different header sets must not share a cached entry. - Broaden `HAS_POST_CONTENT_RE` to match Embed/EmbeddedMedia/EmbeddedMediaImage as tokens inside any class list (single- or double-quoted), and accept single-quoted shell sentinels. Addresses CodeRabbit feedback on #806. --- .../src/runtime/server/instagram-embed.ts | 14 ++++++++++++-- .../runtime/server/utils/instagram-embed.ts | 6 ++++-- test/unit/instagram-embed.test.ts | 18 ++++++++++++++++++ 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/packages/script/src/runtime/server/instagram-embed.ts b/packages/script/src/runtime/server/instagram-embed.ts index 9de193c8f..3d419abb2 100644 --- a/packages/script/src/runtime/server/instagram-embed.ts +++ b/packages/script/src/runtime/server/instagram-embed.ts @@ -26,11 +26,21 @@ const cachedEmbedFetch = defineCachedFunction( return html }, { - name: 'nuxt-scripts-instagram-embed', + // v2 — bump to evict any v1 entries that cached the empty JS shell + // before the shell-detection / UA fix landed. + name: 'nuxt-scripts-instagram-embed-v2', maxAge: 600, swr: true, staleMaxAge: 600, - getKey: (url: string) => url, + // Vary on headers too — Instagram's response is UA-dependent, so + // different callers (e.g. unit tests, future UA changes) must not + // collide on the same key. + getKey: (url: string, headers: Record) => { + const parts = [url] + for (const [k, v] of Object.entries(headers).sort(([a], [b]) => a.localeCompare(b))) + parts.push(`${k}=${v}`) + return parts.join('|') + }, }, ) diff --git a/packages/script/src/runtime/server/utils/instagram-embed.ts b/packages/script/src/runtime/server/utils/instagram-embed.ts index ddf9c8c96..eb5031f4d 100644 --- a/packages/script/src/runtime/server/utils/instagram-embed.ts +++ b/packages/script/src/runtime/server/utils/instagram-embed.ts @@ -5,8 +5,10 @@ export const RSRC_RE = /url\(\/rsrc\.php([^)]+)\)/g // Instagram serves a JS-only shell (splash-screen + comet sentinel, no SSR'd // post markup) when it can't or won't render server-side — e.g. for bot UAs // it can't verify, or for removed/private posts. -const SHELL_BODY_RE = /id="(?:splash-screen|has-finished-comet-page)"/ -const HAS_POST_CONTENT_RE = /class="(?:Embed|EmbeddedMedia)"/ +const SHELL_BODY_RE = /id=["'](?:splash-screen|has-finished-comet-page)["']/ +// Match Embed / EmbeddedMedia / EmbeddedMediaImage as tokens inside any +// class attribute (single- or double-quoted, multi-class lists). +const HAS_POST_CONTENT_RE = /\bclass=(["'])[^"']*\b(?:Embed|EmbeddedMedia|EmbeddedMediaImage)\b[^"']*\1/i export function isEmbedShell(html: string): boolean { return SHELL_BODY_RE.test(html) && !HAS_POST_CONTENT_RE.test(html) diff --git a/test/unit/instagram-embed.test.ts b/test/unit/instagram-embed.test.ts index 029fa1b35..cdef13d67 100644 --- a/test/unit/instagram-embed.test.ts +++ b/test/unit/instagram-embed.test.ts @@ -324,4 +324,22 @@ describe('instagram-embed: isEmbedShell', () => { it('returns false on unrelated HTML (no shell sentinels)', () => { expect(isEmbedShell('nothing here')).toBe(false) }) + + it('detects post content inside multi-class lists', () => { + // Real Instagram responses include splash-screen alongside the SSR'd post; + // multi-class lists like `class="post EmbeddedMedia foo"` must count. + const html = '
' + expect(isEmbedShell(html)).toBe(false) + }) + + it('detects post content with single-quoted class attribute', () => { + const html = `
` + expect(isEmbedShell(html)).toBe(false) + }) + + it('does not match Embed inside an unrelated class token (word boundary)', () => { + // `EmbedSomething` should not count as post content. + const html = '
' + expect(isEmbedShell(html)).toBe(true) + }) })