fix: get external resource blocked (#12927)

## Fix
- Use `[Config].upload.skipSafeFetch` to allow specific external urls
- Use `[Config].upload.pasteURL.allowList` to allow specific external
urls
Documentation: [Uploading files from remote
urls](https://payloadcms.com/docs/upload/overview#uploading-files-from-remote-urls)
Fixes: #12876
Mentioned: #7037,
#12934
Source PR: #12622
Issue Trace:
1. [`allowList`
Added](8b7f2dd#diff-92acf7b8d30e447a791e37820136bcbf23c42f0358daca0fdea4e7b77f7d4bc9
)
2. [`allowList`
Removed](648c168#diff-92acf7b8d30e447a791e37820136bcbf23c42f0358daca0fdea4e7b77f7d4bc9)

Author: kendelljoseph

docs/upload/overview.mdx

@@ -109,6 +109,7 @@ _An asterisk denotes that an option is required._
 |**`mimeTypes`**| Restrict mimeTypes in the file picker. Array of valid mimetypes or mimetype wildcards [More](#mimetypes)|
 |**`pasteURL`**| Controls whether files can be uploaded from remote URLs by pasting them into the Upload field. **Enabled by default.** Accepts `false` to disable or an object with an `allowList` of valid remote URLs. [More](#uploading-files-from-remote-urls)|
 |**`resizeOptions`**| An object passed to the the Sharp image library to resize the uploaded file. [More](https://sharp.pixelplumbing.com/api-resize)|
+|**`skipSafeFetch`**| Set to an `allowList` to skip the safe fetch check when fetching external files. Set to `true` to skip the safe fetch for all documents in this collection. Defaults to `false`. |
 |**`staticDir`**| The folder directory to use to store media in. Can be either an absolute path or relative to the directory that contains your config. Defaults to your collection slug |
 |**`trimOptions`**| An object passed to the the Sharp image library to trim the uploaded file. [More](https://sharp.pixelplumbing.com/api-resize#trim)|
 |**`withMetadata`**| If specified, appends metadata to the output image file. Accepts a boolean or a function that receives `metadata` and `req`, returning a boolean. |
@@ -435,6 +436,24 @@ export const Media: CollectionConfig = {
 }
 ```
 
+You can also adjust server-side fetching at the upload level as well, this does not effect the `CORS` policy like the `pasteURL` option does, but it allows you to skip the safe fetch check for specific URLs.
+
+```
+import type { CollectionConfig } from 'payload'
+
+export const Media: CollectionConfig = {
+ slug: 'media',
+ upload: {
+ skipSafeFetch: [
+ {
+ hostname: 'example.com',
+ pathname: '/images/*',
+ },
+ ],
+ },
+}
+```
+
 ##### Accepted Values for `pasteURL`
 
 | Option | Description |

packages/payload/src/uploads/getExternalFile.ts

@@ -2,6 +2,7 @@ import type { PayloadRequest } from '../types/index.js'
 importtype{File,FileData,UploadConfig}from'./types.js'
 
 import{APIError}from'../errors/index.js'
+import{isURLAllowed}from'../utilities/isURLAllowed.js'
 import{safeFetch}from'./safeFetch.js'
 
 typeArgs={
@@ -23,11 +24,35 @@ export const getExternalFile = async ({ data, req, uploadConfig }: Args): Promis
  ? uploadConfig.externalFileHeaderFilter(Object.fromEntries(newHeaders(req.headers)))
  : {cookie: req.headers.get('cookie')!}
 
-constres=awaitsafeFetch(fileURL,{
-credentials: 'include',
- headers,
-method: 'GET',
-})
+// Check if URL is allowed because of skipSafeFetch allowList
+constskipSafeFetch: boolean=
+uploadConfig.skipSafeFetch===true
+ ? uploadConfig.skipSafeFetch
+ : Array.isArray(uploadConfig.skipSafeFetch)&&
+isURLAllowed(fileURL,uploadConfig.skipSafeFetch)
+
+// Check if URL is allowed because of pasteURL allowList
+constisAllowedPasteUrl: boolean|undefined=
+uploadConfig.pasteURL&&
+uploadConfig.pasteURL.allowList&&
+isURLAllowed(fileURL,uploadConfig.pasteURL.allowList)
+
+letres
+if(skipSafeFetch||isAllowedPasteUrl){
+// Allowed
+res=awaitfetch(fileURL,{
+credentials: 'include',
+ headers,
+method: 'GET',
+})
+}else{
+// Default
+res=awaitsafeFetch(fileURL,{
+credentials: 'include',
+ headers,
+method: 'GET',
+})
+}
 
 if(!res.ok){
 thrownewAPIError(`Failed to fetch file from ${fileURL}`,res.status)

packages/payload/src/uploads/types.ts

@@ -225,7 +225,8 @@ export type UploadConfig = {
 /**
  * Controls the behavior of pasting/uploading files from URLs.
  * If set to `false`, fetching from remote URLs is disabled.
- * If an allowList is provided, server-side fetching will be enabled for specified URLs.
+ * If an `allowList` is provided, server-side fetching will be enabled for specified URLs.
+ *
  * @default true (client-side fetching enabled)
  */
 pasteURL?:
@@ -239,6 +240,11 @@ export type UploadConfig = {
  * @default undefined
  */
 resizeOptions?: ResizeOptions
+/**
+ * Skip safe fetch when using server-side fetching for external files from these URLs.
+ * @default false
+ */
+skipSafeFetch?: AllowList|boolean
 /**
  * The directory to serve static files from. Defaults to collection slug.
  * @default undefined

packages/plugin-cloud-storage/src/plugin.ts

@@ -1,6 +1,6 @@
 importtype{Config}from'payload'
 
-importtype{PluginOptions}from'./types.js'
+importtype{AllowList,PluginOptions}from'./types.js'
 
 import{getFields}from'./fields/getFields.js'
 import{getAfterDeleteHook}from'./hooks/afterDelete.js'
@@ -70,6 +70,47 @@ export const cloudStoragePlugin =
 })
 }
 
+constgetSkipSafeFetchSetting=(): AllowList|boolean=>{
+if(options.disablePayloadAccessControl){
+returntrue
+}
+constisBooleanTrueSkipSafeFetch=
+typeofexistingCollection.upload==='object'&&
+existingCollection.upload.skipSafeFetch===true
+
+constisAllowListSkipSafeFetch=
+typeofexistingCollection.upload==='object'&&
+Array.isArray(existingCollection.upload.skipSafeFetch)
+
+if(isBooleanTrueSkipSafeFetch){
+returntrue
+}elseif(isAllowListSkipSafeFetch){
+constexistingSkipSafeFetch=
+typeofexistingCollection.upload==='object'&&
+Array.isArray(existingCollection.upload.skipSafeFetch)
+ ? existingCollection.upload.skipSafeFetch
+ : []
+
+consthasExactLocalhostMatch=existingSkipSafeFetch.some((entry)=>{
+constentryKeys=Object.keys(entry)
+returnentryKeys.length===1&&entry.hostname==='localhost'
+})
+
+constlocalhostEntry=
+process.env.NODE_ENV!=='production'&&!hasExactLocalhostMatch
+ ? [{hostname: 'localhost'}]
+ : []
+
+return[...existingSkipSafeFetch, ...localhostEntry]
+}
+
+if(process.env.NODE_ENV!=='production'){
+return[{hostname: 'localhost'}]
+}
+
+returnfalse
+}
+
 return{
  ...existingCollection,
  fields,
@@ -92,6 +133,7 @@ export const cloudStoragePlugin =
  ? options.disableLocalStorage
  : true,
  handlers,
+skipSafeFetch: getSkipSafeFetchSetting(),
 },
 }
 }

packages/plugin-cloud-storage/src/types.ts

@@ -81,6 +81,14 @@ export interface GeneratedAdapter {
 
 exporttypeAdapter=(args: {collection: CollectionConfig;prefix?: string})=>GeneratedAdapter
 
+exporttypeAllowList=Array<{
+hostname: string
+pathname?: string
+port?: string
+protocol?: 'http'|'https'
+search?: string
+}>
+
 exporttypeGenerateFileURL=(args: {
 collection: CollectionConfig
 filename: string

test/uploads/config.ts

@@ -1,5 +1,3 @@
-/* eslint-disable no-restricted-exports */
-
 importtype{CollectionSlug,File}from'payload'
 
 importpathfrom'path'
@@ -33,6 +31,7 @@ import {
 reduceSlug,
 relationPreviewSlug,
 relationSlug,
+skipSafeFetchMediaSlug,
 threeDimensionalSlug,
 unstoredMediaSlug,
 versionSlug,
@@ -429,6 +428,14 @@ export default buildConfigWithDefaults({
 staticDir: path.resolve(dirname,'./media'),
 },
 },
+{
+slug: skipSafeFetchMediaSlug,
+fields: [],
+upload: {
+skipSafeFetch: true,
+staticDir: path.resolve(dirname,'./media'),
+},
+},
 {
 slug: animatedTypeMedia,
 fields: [],

test/uploads/int.spec.ts

@@ -1,4 +1,4 @@
-importtype{Payload}from'payload'
+importtype{CollectionSlug,Payload}from'payload'
 
 importfsfrom'fs'
 importpathfrom'path'
@@ -19,6 +19,7 @@ import {
 mediaSlug,
 reduceSlug,
 relationSlug,
+skipSafeFetchMediaSlug,
 unstoredMediaSlug,
 usersSlug,
 }from'./shared.js'
@@ -585,6 +586,22 @@ describe('Collections - Uploads', () => {
 )
 },
 )
+it('should fetch when skipSafeFetch is enabled',async()=>{
+awaitexpect(
+payload.create({
+collection: skipSafeFetchMediaSlugasCollectionSlug,
+data: {
+filename: 'test.png',
+url: 'http://127.0.0.1/file.png',
+},
+}),
+).rejects.toThrow(
+expect.objectContaining({
+name: 'FileRetrievalError',
+message: expect.not.stringContaining('unsafe'),
+}),
+)
+})
 })
 })
 

test/uploads/shared.ts

@@ -25,7 +25,7 @@ export const withoutMetadataSlug = 'without-meta-data'
 exportconstwithOnlyJPEGMetadataSlug='with-only-jpeg-meta-data'
 exportconstcustomFileNameMediaSlug='custom-file-name-media'
 exportconstallowListMediaSlug='allow-list-media'
-
+exportconstskipSafeFetchMediaSlug='skip-safe-fetch-media'
 exportconstlistViewPreviewSlug='list-view-preview'
 exportconstthreeDimensionalSlug='three-dimensional'
 exportconstconstructorOptionsSlug='constructor-options'