fix(seo): noindex non-production sim.ai hosts, redirect indexed 404s

apps/sim/lib/core/utils/urls.test.ts

@@ -17,6 +17,7 @@ import {
  getBrowserOrigin,
  getSocketUrl,
  isLocalhostUrl,
+ isNonCanonicalSimHost,
  isSafeHttpUrl,
  parseOriginList,
 } from '@/lib/core/utils/urls'
@@ -163,3 +164,41 @@ describe('isSafeHttpUrl', () => {
  expect(isSafeHttpUrl('http://')).toBe(false)
  })
 })
+
+describe('isNonCanonicalSimHost', () => {
+ it.each(['www.sim.ai', 'sim.ai', 'WWW.SIM.AI', 'www.sim.ai:443'])(
+ 'treats %s as the canonical marketing site',
+ (host) => {
+ expect(isNonCanonicalSimHost(host)).toBe(false)
+ }
+ )
+
+ it.each(['dev.sim.ai', 'www.dev.sim.ai', 'staging.sim.ai', 'prod.sockets.sim.ai'])(
+ 'treats %s as non-canonical',
+ (host) => {
+ expect(isNonCanonicalSimHost(host)).toBe(true)
+ }
+ )
+
+ it.each(['sim.example.com', 'localhost:3000', 'notsim.ai', 'sim.ai.evil.com'])(
+ 'leaves %s alone',
+ (host) => {
+ expect(isNonCanonicalSimHost(host)).toBe(false)
+ }
+ )
+
+ it.each(['www.sim.ai, dev.sim.ai', 'sim.ai,dev.sim.ai', ' www.sim.ai , staging.sim.ai'])(
+ 'classifies a comma-joined forwarded host by its first entry (%s)',
+ (host) => {
+ expect(isNonCanonicalSimHost(host)).toBe(false)
+ }
+ )
+
+ it('still flags a comma-joined host whose first entry is non-canonical', () => {
+ expect(isNonCanonicalSimHost('dev.sim.ai, www.sim.ai')).toBe(true)
+ })
+
+ it('does not throw on an empty host', () => {
+ expect(isNonCanonicalSimHost('')).toBe(false)
+ })
+})

apps/sim/lib/core/utils/urls.ts

@@ -4,6 +4,9 @@ import { isProd } from '@/lib/core/config/env-flags'
 /** Canonical base URL for the public-facing marketing site. No trailing slash. */
 export const SITE_URL = 'https://www.sim.ai'
 
+/** Host of the canonical marketing site, e.g. `www.sim.ai`. */
+export const CANONICAL_SITE_HOST = new URL(SITE_URL).host
+
 function hasHttpProtocol(url: string): boolean {
  return /^https?:\/\//i.test(url)
 }
@@ -88,14 +91,36 @@ export function getBaseDomain(): string {
  }
 }
 
+/** Drops a leading `www.` label, e.g. `www.sim.ai` -> `sim.ai`. */
+function stripWwwPrefix(host: string): string {
+ return host.startsWith('www.') ? host.slice(4) : host
+}
+
+/**
+ * True for a sim.ai host that is not the canonical marketing site — dev.sim.ai,
+ * staging.sim.ai, and their www variants serve the same build as www.sim.ai, so
+ * search engines treat them as duplicates unless told otherwise.
+ *
+ * `sim.ai` and `www.sim.ai` are both canonical. Self-hosted domains return
+ * false, as do lookalikes such as `notsim.ai`.
+ *
+ * Takes the first entry of a comma-joined forwarded host so a chained proxy
+ * can't make the canonical site look non-canonical via a trailing entry.
+ */
+export function isNonCanonicalSimHost(host: string): boolean {
+ const first = host.split(',')[0]?.trim() ?? ''
+ const hostname = stripWwwPrefix(first.toLowerCase().split(':')[0])
+ const canonical = stripWwwPrefix(CANONICAL_SITE_HOST)
+ return hostname !== canonical && hostname.endsWith(`.${canonical}`)
+}
+
 /**
  * Returns the domain for email addresses, stripping www subdomain for Resend compatibility
  * @returns The email domain (e.g., 'sim.ai' instead of 'www.sim.ai')
  */
 export function getEmailDomain(): string {
  try {
- const baseDomain = getBaseDomain()
- return baseDomain.startsWith('www.') ? baseDomain.substring(4) : baseDomain
+ return stripWwwPrefix(getBaseDomain())
  } catch (_e) {
  return isProd ? 'sim.ai' : 'localhost:3000'
  }

apps/sim/next.config.ts

@@ -531,6 +531,28 @@ const nextConfig: NextConfig = {
  }
  )
 
+ /**
+ * Indexed 404s from an external SEO audit. The capability paths read as
+ * tool/feature pages and map to the integrations catalog; the rest have no
+ * closer successor than the homepage.
+ *
+ * `/security` is deliberately excluded: security.txt advertises it as the
+ * RFC 9116 `Policy` URI, so a permanent redirect to marketing would both
+ * mislead that link and shadow a real policy page added later.
+ */
+ redirects.push(
+ ...['read', 'research', 'scrape'].map((slug) => ({
+ source: `/${slug}`,
+ destination: '/integrations',
+ permanent: true,
+ })),
+ ...['actions', 'crawl', 'fast'].map((slug) => ({
+ source: `/${slug}`,
+ destination: '/',
+ permanent: true,
+ }))
+ )
+
  return redirects
  },
  async rewrites() {

apps/sim/proxy.ts

@@ -6,6 +6,7 @@ import { getEnv } from './lib/core/config/env'
 import { isAuthDisabled, isDev, isHosted } from './lib/core/config/env-flags'
 import { generateRuntimeCSP } from './lib/core/security/csp'
 import { getClientIp } from './lib/core/utils/request'
+import { isNonCanonicalSimHost } from './lib/core/utils/urls'
 
 const logger = createLogger('Proxy')
 
@@ -297,10 +298,31 @@ export async function proxy(request: NextRequest) {
  return track(request, response)
 }
 
+/**
+ * Keeps non-production sim.ai deployments out of search results.
+ *
+ * `noindex` rather than a robots.txt `Disallow` is deliberate: a disallowed URL
+ * can still be indexed when linked externally, and blocking the crawl stops
+ * search engines from ever seeing the directive that removes pages already in
+ * the index. robots.txt is excluded from this proxy's matcher so it keeps
+ * serving the crawlable rules this header depends on.
+ */
+function applyIndexingPolicy(request: NextRequest, response: NextResponse): void {
+ const host =
+ request.headers.get('x-forwarded-host')?.split(',')[0]?.trim() ||
+ request.headers.get('host') ||
+ request.nextUrl.host
+
+ if (isNonCanonicalSimHost(host)) {
+ response.headers.set('X-Robots-Tag', 'noindex, nofollow')
+ }
+}
+
 /**
  * Sends request data to Profound analytics (fire-and-forget) and returns the response.
  */
 function track(request: NextRequest, response: NextResponse): NextResponse {
+ applyIndexingPolicy(request, response)
  sendToProfound(request, response.status)
  return response
 }

packages/testing/src/mocks/urls.mock.ts

@@ -10,6 +10,9 @@ export const LOCALHOST_HOSTNAMES_MOCK: ReadonlySet<string> = new Set([
  '::1',
 ])
 
+/** Mirrors the real `CANONICAL_SITE_HOST` from `@/lib/core/utils/urls`. */
+export const CANONICAL_SITE_HOST_MOCK = 'www.sim.ai'
+
 const DEFAULT_SOCKET_URL = 'http://localhost:3002'
 const DEFAULT_OLLAMA_URL = 'http://localhost:11434'
 
@@ -72,6 +75,18 @@ function isLoopbackHostnameImpl(hostname: string): boolean {
  return LOCALHOST_HOSTNAMES_MOCK.has(hostname)
 }
 
+/** Mirrors the real `stripWwwPrefix` from `@/lib/core/utils/urls`. */
+function stripWwwPrefix(host: string): string {
+ return host.startsWith('www.') ? host.slice(4) : host
+}
+
+function isNonCanonicalSimHostImpl(host: string): boolean {
+ const first = host.split(',')[0]?.trim() ?? ''
+ const hostname = stripWwwPrefix(first.toLowerCase().split(':')[0])
+ const canonical = stripWwwPrefix(CANONICAL_SITE_HOST_MOCK)
+ return hostname !== canonical && hostname.endsWith(`.${canonical}`)
+}
+
 function parseOriginListImpl(
  raw: string | undefined | null,
  onInvalid?: (value: string) => void
@@ -156,6 +171,7 @@ export const urlsMockFns = {
  mockGetBaseDomain: vi.fn(getBaseDomainImpl),
  mockGetEmailDomain: vi.fn(getEmailDomainImpl),
  mockIsLoopbackHostname: vi.fn(isLoopbackHostnameImpl),
+ mockIsNonCanonicalSimHost: vi.fn(isNonCanonicalSimHostImpl),
  mockParseOriginList: vi.fn(parseOriginListImpl),
  mockIsLocalhostUrl: vi.fn(isLocalhostUrlImpl),
  mockGetBrowserOrigin: vi.fn(getBrowserOriginImpl),
@@ -176,6 +192,7 @@ export function resetUrlsMock(): void {
  urlsMockFns.mockGetBaseDomain.mockReset().mockImplementation(getBaseDomainImpl)
  urlsMockFns.mockGetEmailDomain.mockReset().mockImplementation(getEmailDomainImpl)
  urlsMockFns.mockIsLoopbackHostname.mockReset().mockImplementation(isLoopbackHostnameImpl)
+ urlsMockFns.mockIsNonCanonicalSimHost.mockReset().mockImplementation(isNonCanonicalSimHostImpl)
  urlsMockFns.mockParseOriginList.mockReset().mockImplementation(parseOriginListImpl)
  urlsMockFns.mockIsLocalhostUrl.mockReset().mockImplementation(isLocalhostUrlImpl)
  urlsMockFns.mockGetBrowserOrigin.mockReset().mockImplementation(getBrowserOriginImpl)
@@ -197,12 +214,14 @@ export function resetUrlsMock(): void {
 export const urlsMock = {
  SITE_URL: 'https://www.sim.ai',
  LOCALHOST_HOSTNAMES: LOCALHOST_HOSTNAMES_MOCK,
+ CANONICAL_SITE_HOST: CANONICAL_SITE_HOST_MOCK,
  getBaseUrl: urlsMockFns.mockGetBaseUrl,
  getInternalApiBaseUrl: urlsMockFns.mockGetInternalApiBaseUrl,
  ensureAbsoluteUrl: urlsMockFns.mockEnsureAbsoluteUrl,
  getBaseDomain: urlsMockFns.mockGetBaseDomain,
  getEmailDomain: urlsMockFns.mockGetEmailDomain,
  isLoopbackHostname: urlsMockFns.mockIsLoopbackHostname,
+ isNonCanonicalSimHost: urlsMockFns.mockIsNonCanonicalSimHost,
  parseOriginList: urlsMockFns.mockParseOriginList,
  isLocalhostUrl: urlsMockFns.mockIsLocalhostUrl,
  getBrowserOrigin: urlsMockFns.mockGetBrowserOrigin,