fix(instagram-embed): use facebookexternalhit UA & don't cache empty shells

packages/script/src/runtime/server/instagram-embed.ts

@@ -1,8 +1,9 @@
 import { createError, defineEventHandler, getQuery, setHeader } from 'h3'
-import { useRuntimeConfig } from 'nitropack/runtime'
+import { defineCachedFunction, useRuntimeConfig } from 'nitropack/runtime'
+import { $fetch } from 'ofetch'
 import { ELEMENT_NODE, parse, renderSync, TEXT_NODE, walkSync } from 'ultrahtml'
 import { createCachedJsonFetch } from './utils/cached-upstream'
-import { proxyAssetUrl, rewriteUrl, rewriteUrlsInText, RSRC_RE, scopeCss } from './utils/instagram-embed'
+import { isEmbedShell, proxyAssetUrl, rewriteUrl, rewriteUrlsInText, RSRC_RE, scopeCss } from './utils/instagram-embed'
 import { withSigning } from './utils/withSigning'
 
 export { proxyAssetUrl, proxyImageUrl, rewriteUrl, rewriteUrlsInText, scopeCss } from './utils/instagram-embed'
@@ -12,10 +13,35 @@ const SRCSET_SPLIT_RE = /\s+/
 
 // Instagram embed HTML is semi-fresh (likes, captions may update); 10min
 // matches the outbound Cache-Control header and dedupes per post+captions.
-const cachedEmbedFetch = createCachedJsonFetch<string>(
- 'nuxt-scripts-instagram-embed',
- 600,
- url => url,
+// Throws on shell responses so nitro doesn't cache them.
+const cachedEmbedFetch = defineCachedFunction(
+ async (url: string, headers: Record<string, string>): Promise<string> => {
+ const html = await $fetch<string>(url, { timeout: 10000, headers })
+ if (isEmbedShell(html)) {
+ throw createError({
+ statusCode: 502,
+ statusMessage: 'Instagram returned an empty embed shell (post unavailable or upstream rate-limiting)',
+ })
+ }
+ return html
+ },
+ {
+ // v2 — bump to evict any v1 entries that cached the empty JS shell
+ // before the shell-detection / UA fix landed.
+ name: 'nuxt-scripts-instagram-embed-v2',
+ maxAge: 600,
+ swr: true,
+ staleMaxAge: 600,
+ // Vary on headers too — Instagram's response is UA-dependent, so
+ // different callers (e.g. unit tests, future UA changes) must not
+ // collide on the same key.
+ getKey: (url: string, headers: Record<string, string>) => {
+ const parts = [url]
+ for (const [k, v] of Object.entries(headers).sort(([a], [b]) => a.localeCompare(b)))
+ parts.push(`${k}=${v}`)
+ return parts.join('|')
+ },
+ },
 )
 
 // Static CSS from Instagram's CDN is versioned; 24h cache is safe because the
@@ -79,10 +105,12 @@ export default withSigning(defineEventHandler(async (event) => {
  const embedUrl = `${cleanUrl}embed/${captions ? 'captioned/' : ''}`
 
  const html = await cachedEmbedFetch(embedUrl, {
- headers: {
- 'Accept': 'text/html',
- 'User-Agent': 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
- },
+ 'Accept': 'text/html',
+ // Meta's own crawler UA. Googlebot's UA is also accepted by Instagram
+ // but is IP-verified, so it fails from hosts outside Google's ranges
+ // (e.g. Cloudflare/Vercel) and Instagram serves the JS shell instead
+ // of the SSR'd post.
+ 'User-Agent': 'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)',
  }).catch((error: any) => {
  throw createError({
  statusCode: error.statusCode || 500,

packages/script/src/runtime/server/utils/instagram-embed.ts

@@ -1,6 +1,19 @@
 import { buildProxyUrl } from './proxy-url'
 
 export const RSRC_RE = /url\(\/rsrc\.php([^)]+)\)/g
+
+// Instagram serves a JS-only shell (splash-screen + comet sentinel, no SSR'd
+// post markup) when it can't or won't render server-side — e.g. for bot UAs
+// it can't verify, or for removed/private posts.
+const SHELL_BODY_RE = /id=["'](?:splash-screen|has-finished-comet-page)["']/
+// Match Embed / EmbeddedMedia / EmbeddedMediaImage as tokens inside any
+// class attribute (single- or double-quoted, multi-class lists).
+const HAS_POST_CONTENT_RE = /\bclass=(["'])[^"']*\b(?:Embed|EmbeddedMedia|EmbeddedMediaImage)\b[^"']*\1/i
+
+export function isEmbedShell(html: string): boolean {
+ return SHELL_BODY_RE.test(html) && !HAS_POST_CONTENT_RE.test(html)
+}
+
 export const AMP_RE = /&/g
 export const SCONTENT_RE = /https:\/\/scontent[^"'\s),]+\.cdninstagram\.com[^"'\s),]+/g
 export const STATIC_CDN_RE = /https:\/\/static\.cdninstagram\.com[^"'\s),]+/g

test/unit/instagram-embed.test.ts

@@ -1,6 +1,7 @@
 import { ELEMENT_NODE, parse, renderSync, TEXT_NODE, walkSync } from 'ultrahtml'
 import { describe, expect, it } from 'vitest'
 import {
+ isEmbedShell,
  proxyImageUrl,
  rewriteUrl,
  rewriteUrlsInText,
@@ -302,3 +303,43 @@ html, body { margin: 0; padding: 0; }
  expect(result).toContain(`${scope} [data-x="a,b"] .Embed`)
  })
 })
+
+describe('instagram-embed: isEmbedShell', () => {
+ it('detects JS-only shell with splash-screen and no post markup', () => {
+ const html = '<body><div id="splash-screen"></div><div id="has-finished-comet-page"></div></body>'
+ expect(isEmbedShell(html)).toBe(true)
+ })
+
+ it('accepts real SSR\'d post even when shell sentinels appear elsewhere', () => {
+ // Some Instagram responses include the comet sentinel alongside real content.
+ const html = '<body><a class="EmbeddedMedia"><img class="EmbeddedMediaImage" /></a><div id="has-finished-comet-page"></div></body>'
+ expect(isEmbedShell(html)).toBe(false)
+ })
+
+ it('accepts real post with Embed wrapper', () => {
+ const html = '<div class="Embed" data-media-id="123"><div>content</div></div>'
+ expect(isEmbedShell(html)).toBe(false)
+ })
+
+ it('returns false on unrelated HTML (no shell sentinels)', () => {
+ expect(isEmbedShell('<html><body>nothing here</body></html>')).toBe(false)
+ })
+
+ it('detects post content inside multi-class lists', () => {
+ // Real Instagram responses include splash-screen alongside the SSR'd post;
+ // multi-class lists like `class="post EmbeddedMedia foo"` must count.
+ const html = '<div id="splash-screen"></div><div class="post EmbeddedMedia foo"></div>'
+ expect(isEmbedShell(html)).toBe(false)
+ })
+
+ it('detects post content with single-quoted class attribute', () => {
+ const html = `<div id='splash-screen'></div><a class='EmbeddedMedia'></a>`
+ expect(isEmbedShell(html)).toBe(false)
+ })
+
+ it('does not match Embed inside an unrelated class token (word boundary)', () => {
+ // `EmbedSomething` should not count as post content.
+ const html = '<div id="splash-screen"></div><div class="NotAnEmbedThing"></div>'
+ expect(isEmbedShell(html)).toBe(true)
+ })
+})