Text attributes data truncation

src/Database/Adapter/MariaDB.php

@@ -1601,11 +1601,11 @@ protected function getSQLType(string $type, int $size, bool $signed = true, bool
 
  case Database::VAR_STRING:
  // $size = $size * 4; // Convert utf8mb4 size to bytes
- if ($size > 16777215) {
+ if ($size > Database::MAX_MEDIUMTEXT_BYTES) {
  return 'LONGTEXT';
  }
 
- if ($size > 65535) {
+ if ($size > Database::MAX_TEXT_BYTES) {
  return 'MEDIUMTEXT';
  }
 

src/Database/Adapter/SQLite.php

@@ -35,15 +35,6 @@
  */
 class SQLite extends MariaDB
 {
- /**
- * MariaDB byte ceilings for TEXT-family types, mirrored so PRAGMA-based
- * introspection produces the same characterMaximumLength values that
- * INFORMATION_SCHEMA.COLUMNS would on MariaDB.
- */
- private const MARIADB_TEXT_BYTES = '65535';
- private const MARIADB_MEDIUMTEXT_BYTES = '16777215';
- private const MARIADB_LONGTEXT_BYTES = '4294967295';
-
  /** Suffix appended to every FTS5 virtual table name created by this adapter. */
  private const FTS_TABLE_SUFFIX = '_fts';
 
@@ -2973,19 +2964,25 @@ private function parseSqliteColumnType(string $declaration): array
  break;
  }
 
+ /**
+ * MariaDB byte ceilings for TEXT-family types, mirrored so PRAGMA-based
+ * introspection produces the same characterMaximumLength values that
+ * INFORMATION_SCHEMA.COLUMNS would on MariaDB.
+ */
+
  if ($this->emulateMySQL) {
  switch ($dataType) {
  case 'text':
- $result['characterMaximumLength'] = self::MARIADB_TEXT_BYTES;
+ $result['characterMaximumLength'] = '' . Database::MAX_TEXT_BYTES;
  break;
 
  case 'mediumtext':
- $result['characterMaximumLength'] = self::MARIADB_MEDIUMTEXT_BYTES;
+ $result['characterMaximumLength'] = '' . Database::MAX_MEDIUMTEXT_BYTES;
  break;
 
  case 'longtext':
  case 'json':
- $result['characterMaximumLength'] = self::MARIADB_LONGTEXT_BYTES;
+ $result['characterMaximumLength'] = '' . Database::MAX_LONGTEXT_BYTES;
  break;
 
  case 'tinyint':

src/Database/Database.php

@@ -115,6 +115,11 @@ class Database
  public const MAX_ARRAY_INDEX_LENGTH = 255;
  public const MAX_UID_DEFAULT_LENGTH = 36;
 
+ // Maximum byte capacity for TEXT
+ public const MAX_TEXT_BYTES = 65535;
+ public const MAX_MEDIUMTEXT_BYTES = 16777215;
+ public const MAX_LONGTEXT_BYTES = 4294967295;
+
  // Min limits
  public const MIN_INT = -2147483648;
 

src/Database/Validator/Attribute.php

@@ -317,22 +317,22 @@ public function checkType(Document $attribute): bool
  break;
 
  case Database::VAR_TEXT:
- if ($size > 65535) {
- $this->message = 'Max size allowed for text is: 65535';
+ if ($size > Database::MAX_TEXT_BYTES) {
+ $this->message = 'Max size allowed for text is: ' . Database::MAX_TEXT_BYTES;
  throw new DatabaseException($this->message);
  }
  break;
 
  case Database::VAR_MEDIUMTEXT:
- if ($size > 16777215) {
- $this->message = 'Max size allowed for mediumtext is: 16777215';
+ if ($size > Database::MAX_MEDIUMTEXT_BYTES) {
+ $this->message = 'Max size allowed for mediumtext is: ' . Database::MAX_MEDIUMTEXT_BYTES;
  throw new DatabaseException($this->message);
  }
  break;
 
  case Database::VAR_LONGTEXT:
- if ($size > 4294967295) {
- $this->message = 'Max size allowed for longtext is: 4294967295';
+ if ($size > Database::MAX_LONGTEXT_BYTES) {
+ $this->message = 'Max size allowed for longtext is: ' . Database::MAX_LONGTEXT_BYTES;
  throw new DatabaseException($this->message);
  }
  break;

src/Database/Validator/ByteLength.php

@@ -0,0 +1,53 @@
+<?php
+
+namespace Utopia\Database\Validator;
+
+use Utopia\Validator;
+
+/**
+ * ByteLength
+ *
+ * Validate that a string's byte length does not exceed a maximum. Unlike the
+ * character-based Text validator, this measures the actual stored size, which
+ * is what byte-capacity columns (e.g. MySQL/MariaDB TEXT family) are limited by.
+ */
+class ByteLength extends Validator
+{
+ protected int $max;
+
+ /**
+ * @param int $max Maximum allowed byte length. Use 0 for unlimited.
+ */
+ public function __construct(int $max)
+ {
+ $this->max = $max;
+ }
+
+ public function getDescription(): string
+ {
+ return 'Value must be a valid string no longer than ' . $this->max . ' bytes';
+ }
+
+ public function isArray(): bool
+ {
+ return false;
+ }
+
+ public function getType(): string
+ {
+ return self::TYPE_STRING;
+ }
+
+ public function isValid(mixed $value): bool
+ {
+ if (!\is_string($value)) {
+ return false;
+ }
+
+ if ($this->max !== 0 && \strlen($value) > $this->max) {
+ return false;
+ }
+
+ return true;
+ }
+}

src/Database/Validator/Structure.php

@@ -353,10 +353,22 @@ protected function checkForInvalidAttributeValues(Document $document, array $str
  $validators[] = new Sequence($this->idAttributeType, $attribute['$id'] === '$sequence');
  break;
 
- case Database::VAR_VARCHAR:
  case Database::VAR_TEXT:
+ $validators[] = new ByteLength($size);
+ $validators[] = new ByteLength(Database::MAX_TEXT_BYTES);
+ break;
+
  case Database::VAR_MEDIUMTEXT:
+ $validators[] = new ByteLength($size);
+ $validators[] = new ByteLength(Database::MAX_MEDIUMTEXT_BYTES);
+ break;
+
  case Database::VAR_LONGTEXT:
+ $validators[] = new ByteLength($size);
+ $validators[] = new ByteLength(Database::MAX_LONGTEXT_BYTES);
+ break;
+
+ case Database::VAR_VARCHAR:
  case Database::VAR_STRING:
  $validators[] = new Text($size, min: 0);
  break;

tests/e2e/Adapter/Scopes/DocumentTests.php

@@ -971,6 +971,125 @@ public function testUpsertDocuments(): void
  }
  }
 
+ public function testTextByteTruncationCreate(): void
+ {
+ /** @var Database $database */
+ $database = $this->getDatabase();
+
+ // Byte-capacity validation relies on attribute metadata, which
+ // schemaless adapters don't store, so there is nothing to enforce.
+ if (!$database->getAdapter()->getSupportForAttributes()) {
+ $this->expectNotToPerformAssertions();
+ return;
+ }
+
+ $database->createCollection(__FUNCTION__);
+
+ // A `text` attribute at its maximum allowed size. On MySQL/MariaDB this
+ // maps to a TEXT column, which is limited to 65,535 *bytes*.
+ $database->createAttribute(__FUNCTION__, 'text', Database::VAR_TEXT, 65535, false);
+
+ // The Structure validator caps a TEXT column at its 65,535-byte capacity,
+ // measuring the value's actual byte length. A 20,000-char emoji value is
+ // 80,000 bytes (4 bytes per char in utf8mb4), so it exceeds the column's
+ // byte capacity and must be rejected up front with a clean
+ // StructureException, rather than letting the database raise error 1406
+ // (data truncation).
+ $value = \str_repeat('📝', 20000);
+ $this->assertGreaterThan(65535, \strlen($value)); // exceeds the byte capacity
+
+ $document = new Document([
+ '$id' => 'first',
+ 'text' => $value,
+ '$permissions' => [
+ Permission::read(Role::any()),
+ Permission::create(Role::any()),
+ Permission::update(Role::any()),
+ Permission::delete(Role::any()),
+ ],
+ ]);
+
+ try {
+ $database->createDocument(__FUNCTION__, $document);
+ $this->fail('Expected StructureException for over-capacity text value');
+ } catch (StructureException $e) {
+ $this->assertStringContainsString('65535 bytes', $e->getMessage());
+ }
+ }
+
+ public function testTextByteTruncationValid(): void
+ {
+ /** @var Database $database */
+ $database = $this->getDatabase();
+
+ if (!$database->getAdapter()->getSupportForAttributes()) {
+ $this->expectNotToPerformAssertions();
+ return;
+ }
+
+ $database->createCollection(__FUNCTION__);
+ $database->createAttribute(__FUNCTION__, 'text', Database::VAR_TEXT, 65535, false);
+
+ // A value that fills the column's full byte capacity is stored and
+ // round-trips intact. 65,535 ASCII chars are exactly 65,535 bytes, so
+ // byte-based validation accepts the whole column, where the previous
+ // char-based cap would have rejected anything over 16,383 chars.
+ $okValue = \str_repeat('a', 65535);
+
+ $document = new Document([
+ '$id' => 'first',
+ 'text' => $okValue,
+ '$permissions' => [
+ Permission::read(Role::any()),
+ Permission::create(Role::any()),
+ Permission::update(Role::any()),
+ Permission::delete(Role::any()),
+ ],
+ ]);
+
+ $created = $database->createDocument(__FUNCTION__, $document);
+ $fetched = $database->getDocument(__FUNCTION__, $created->getId());
+ $this->assertEquals($okValue, $fetched->getAttribute('text'));
+ }
+
+ public function testTextByteTruncationUpdate(): void
+ {
+ /** @var Database $database */
+ $database = $this->getDatabase();
+
+ if (!$database->getAdapter()->getSupportForAttributes()) {
+ $this->expectNotToPerformAssertions();
+ return;
+ }
+
+ $database->createCollection(__FUNCTION__);
+ $database->createAttribute(__FUNCTION__, 'text', Database::VAR_TEXT, 65535, false);
+
+ $document = new Document([
+ '$id' => 'first',
+ 'text' => \str_repeat('a', 16383),
+ '$permissions' => [
+ Permission::read(Role::any()),
+ Permission::create(Role::any()),
+ Permission::update(Role::any()),
+ Permission::delete(Role::any()),
+ ],
+ ]);
+
+ $created = $database->createDocument(__FUNCTION__, $document);
+
+ // An oversized value is rejected on update, the same as on create.
+ $value = \str_repeat('📝', 20000);
+ $this->assertGreaterThan(65535, \strlen($value)); // exceeds the byte capacity
+
+ try {
+ $database->updateDocument(__FUNCTION__, $created->getId(), $created->setAttribute('text', $value));
+ $this->fail('Expected StructureException for over-capacity text value on update');
+ } catch (StructureException $e) {
+ $this->assertStringContainsString('65535 bytes', $e->getMessage());
+ }
+ }
+
  public function testUpsertDocumentsInc(): void
  {
  /** @var Database $database */