fix(instagram-embed): use facebookexternalhit UA & don't cache empty shells (#806)

Author: harlan-zw

packages/script/src/runtime/server/instagram-embed.ts

@@ -1,8 +1,9 @@
 import{createError,defineEventHandler,getQuery,setHeader}from'h3'
-import{useRuntimeConfig}from'nitropack/runtime'
+import{defineCachedFunction,useRuntimeConfig}from'nitropack/runtime'
+import{$fetch}from'ofetch'
 import{ELEMENT_NODE,parse,renderSync,TEXT_NODE,walkSync}from'ultrahtml'
 import{createCachedJsonFetch}from'./utils/cached-upstream'
-import{proxyAssetUrl,rewriteUrl,rewriteUrlsInText,RSRC_RE,scopeCss}from'./utils/instagram-embed'
+import{isEmbedShell,proxyAssetUrl,rewriteUrl,rewriteUrlsInText,RSRC_RE,scopeCss}from'./utils/instagram-embed'
 import{withSigning}from'./utils/withSigning'
 
 export{proxyAssetUrl,proxyImageUrl,rewriteUrl,rewriteUrlsInText,scopeCss}from'./utils/instagram-embed'
@@ -12,10 +13,35 @@ const SRCSET_SPLIT_RE = /\s+/
 
 // Instagram embed HTML is semi-fresh (likes, captions may update); 10min
 // matches the outbound Cache-Control header and dedupes per post+captions.
-constcachedEmbedFetch=createCachedJsonFetch<string>(
-'nuxt-scripts-instagram-embed',
-600,
-url=>url,
+// Throws on shell responses so nitro doesn't cache them.
+constcachedEmbedFetch=defineCachedFunction(
+async(url: string,headers: Record<string,string>): Promise<string>=>{
+consthtml=await$fetch<string>(url,{timeout: 10000, headers })
+if(isEmbedShell(html)){
+throwcreateError({
+statusCode: 502,
+statusMessage: 'Instagram returned an empty embed shell (post unavailable or upstream rate-limiting)',
+})
+}
+returnhtml
+},
+{
+// v2 — bump to evict any v1 entries that cached the empty JS shell
+// before the shell-detection / UA fix landed.
+name: 'nuxt-scripts-instagram-embed-v2',
+maxAge: 600,
+swr: true,
+staleMaxAge: 600,
+// Vary on headers too — Instagram's response is UA-dependent, so
+// different callers (e.g. unit tests, future UA changes) must not
+// collide on the same key.
+getKey: (url: string,headers: Record<string,string>)=>{
+constparts=[url]
+for(const[k,v]ofObject.entries(headers).sort(([a],[b])=>a.localeCompare(b)))
+parts.push(`${k}=${v}`)
+returnparts.join('|')
+},
+},
 )
 
 // Static CSS from Instagram's CDN is versioned; 24h cache is safe because the
@@ -79,10 +105,12 @@ export default withSigning(defineEventHandler(async (event) => {
 constembedUrl=`${cleanUrl}embed/${captions ? 'captioned/' : ''}`
 
 consthtml=awaitcachedEmbedFetch(embedUrl,{
-headers: {
-'Accept': 'text/html',
-'User-Agent': 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
-},
+'Accept': 'text/html',
+// Meta's own crawler UA. Googlebot's UA is also accepted by Instagram
+// but is IP-verified, so it fails from hosts outside Google's ranges
+// (e.g. Cloudflare/Vercel) and Instagram serves the JS shell instead
+// of the SSR'd post.
+'User-Agent': 'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)',
 }).catch((error: any)=>{
 throwcreateError({
 statusCode: error.statusCode||500,

packages/script/src/runtime/server/utils/instagram-embed.ts

@@ -1,6 +1,19 @@
 import{buildProxyUrl}from'./proxy-url'
 
 exportconstRSRC_RE=/url\(\/rsrc\.php([^)]+)\)/g
+
+// Instagram serves a JS-only shell (splash-screen + comet sentinel, no SSR'd
+// post markup) when it can't or won't render server-side — e.g. for bot UAs
+// it can't verify, or for removed/private posts.
+constSHELL_BODY_RE=/id=["'](?:splash-screen|has-finished-comet-page)["']/
+// Match Embed / EmbeddedMedia / EmbeddedMediaImage as tokens inside any
+// class attribute (single- or double-quoted, multi-class lists).
+constHAS_POST_CONTENT_RE=/\bclass=(["'])[^"']*\b(?:Embed|EmbeddedMedia|EmbeddedMediaImage)\b[^"']*\1/i
+
+exportfunctionisEmbedShell(html: string): boolean{
+returnSHELL_BODY_RE.test(html)&&!HAS_POST_CONTENT_RE.test(html)
+}
+
 exportconstAMP_RE=/&/g
 exportconstSCONTENT_RE=/https:\/\/scontent[^"'\s),]+\.cdninstagram\.com[^"'\s),]+/g
 exportconstSTATIC_CDN_RE=/https:\/\/static\.cdninstagram\.com[^"'\s),]+/g

test/unit/instagram-embed.test.ts

@@ -1,6 +1,7 @@
 import{ELEMENT_NODE,parse,renderSync,TEXT_NODE,walkSync}from'ultrahtml'
 import{describe,expect,it}from'vitest'
 import{
+isEmbedShell,
 proxyImageUrl,
 rewriteUrl,
 rewriteUrlsInText,
@@ -302,3 +303,43 @@ html, body { margin: 0; padding: 0; }
 expect(result).toContain(`${scope} [data-x="a,b"] .Embed`)
 })
 })
+
+describe('instagram-embed: isEmbedShell',()=>{
+it('detects JS-only shell with splash-screen and no post markup',()=>{
+consthtml='<body><div id="splash-screen"></div><div id="has-finished-comet-page"></div></body>'
+expect(isEmbedShell(html)).toBe(true)
+})
+
+it('accepts real SSR\'d post even when shell sentinels appear elsewhere',()=>{
+// Some Instagram responses include the comet sentinel alongside real content.
+consthtml='<body><a class="EmbeddedMedia"><img class="EmbeddedMediaImage" /></a><div id="has-finished-comet-page"></div></body>'
+expect(isEmbedShell(html)).toBe(false)
+})
+
+it('accepts real post with Embed wrapper',()=>{
+consthtml='<div class="Embed" data-media-id="123"><div>content</div></div>'
+expect(isEmbedShell(html)).toBe(false)
+})
+
+it('returns false on unrelated HTML (no shell sentinels)',()=>{
+expect(isEmbedShell('<html><body>nothing here</body></html>')).toBe(false)
+})
+
+it('detects post content inside multi-class lists',()=>{
+// Real Instagram responses include splash-screen alongside the SSR'd post;
+// multi-class lists like `class="post EmbeddedMedia foo"` must count.
+consthtml='<div id="splash-screen"></div><div class="post EmbeddedMedia foo"></div>'
+expect(isEmbedShell(html)).toBe(false)
+})
+
+it('detects post content with single-quoted class attribute',()=>{
+consthtml=`<div id='splash-screen'></div><a class='EmbeddedMedia'></a>`
+expect(isEmbedShell(html)).toBe(false)
+})
+
+it('does not match Embed inside an unrelated class token (word boundary)',()=>{
+// `EmbedSomething` should not count as post content.
+consthtml='<div id="splash-screen"></div><div class="NotAnEmbedThing"></div>'
+expect(isEmbedShell(html)).toBe(true)
+})
+})