Make content_ids migration incremental and restartable

CHANGES/+content-ids-migration.bugfix

@@ -0,0 +1 @@
+Filled missing repository version `content_ids` in SQL from content membership, committing per repository so large databases are less likely to hit statement timeouts.

pulpcore/app/migrations/0152_alter_repositoryversion_content_ids.py

@@ -1,35 +1,65 @@
 # Generated by Django 5.2.13 on 2026-06-08 21:54
 
 import django.contrib.postgres.fields
-from django.db import migrations, models
+from django.db import migrations, models, transaction
+
+# Same membership as RepositoryVersion._content_relationships().
+_POPULATE_SQL = """
+UPDATE __RV__ AS rv
+SET content_ids = COALESCE((
+ SELECT ARRAY_AGG(rc.content_id)
+ FROM __RC__ AS rc
+ INNER JOIN __RV__ AS va ON va.pulp_id = rc.version_added_id
+ LEFT JOIN __RV__ AS vr ON vr.pulp_id = rc.version_removed_id
+ WHERE rc.repository_id = rv.repository_id
+ AND va.number <= rv.number
+ AND (vr.pulp_id IS NULL OR vr.number > rv.number)
+), '{}'::uuid[])
+WHERE rv.repository_id = %s
+ AND rv.content_ids IS NULL
+"""
 
 
 def populate_content_ids(apps, schema_editor):
- RepositoryVersion = apps.get_model('core', 'RepositoryVersion')
- RepositoryContent = apps.get_model('core', 'RepositoryContent')
- repo_versions = []
- for rv in RepositoryVersion.objects.filter(content_ids=None).iterator(chunk_size=2000):
- rv.content_ids = list(RepositoryContent.objects.filter(
- repository_id=rv.repository_id, version_added__number__lte=rv.number
- ).exclude(version_removed__number__lte=rv.number).values_list("content_id", flat=True))
- repo_versions.append(rv)
- if len(repo_versions) >= 2000:
- RepositoryVersion.objects.bulk_update(repo_versions, ['content_ids'])
- repo_versions = []
- if repo_versions:
- RepositoryVersion.objects.bulk_update(repo_versions, ['content_ids'])
+ """Fill missing content_ids from content membership, committing per repository."""
+ RepositoryVersion = apps.get_model("core", "RepositoryVersion")
+ RepositoryContent = apps.get_model("core", "RepositoryContent")
+ connection = schema_editor.connection
+ qn = schema_editor.quote_name
+ rv = qn(RepositoryVersion._meta.db_table)
+ rc = qn(RepositoryContent._meta.db_table)
+ sql = _POPULATE_SQL.replace("__RV__", rv).replace("__RC__", rc)
+
+ with connection.cursor() as cursor:
+ cursor.execute(f"SELECT DISTINCT repository_id FROM {rv} WHERE content_ids IS NULL")
+ repo_ids = [row[0] for row in cursor.fetchall()]
+
+ for repo_id in repo_ids:
+ with transaction.atomic(using=connection.alias):
+ with connection.cursor() as cursor:
+ cursor.execute(sql, [repo_id])
+
 
 class Migration(migrations.Migration):
+ # Per-repository commits inside RunPython; AlterField runs afterwards.
+ atomic = False
 
  dependencies = [
- ('core', '0151_upstreampulp_connect_timeout_and_more'),
+ ("core", "0151_upstreampulp_connect_timeout_and_more"),
  ]
 
  operations = [
- migrations.RunPython(populate_content_ids, reverse_code=migrations.RunPython.noop, elidable=True),
+ migrations.RunPython(
+ populate_content_ids,
+ reverse_code=migrations.RunPython.noop,
+ elidable=True,
+ atomic=False,
+ ),
  migrations.AlterField(
- model_name='repositoryversion',
- name='content_ids',
- field=django.contrib.postgres.fields.ArrayField(base_field=models.UUIDField(), default=list, size=None),
+ model_name="repositoryversion",
+ name="content_ids",
+ field=django.contrib.postgres.fields.ArrayField(
+ base_field=models.UUIDField(), default=list, size=None
+ ),
  ),
  ]

pulpcore/tests/unit/models/test_content_ids_migration.py

@@ -0,0 +1,77 @@
+import importlib
+from types import SimpleNamespace
+from uuid import uuid4
+
+import pytest
+from django.apps import apps
+from django.db import connection
+
+from pulpcore.app.models import RepositoryVersion
+from pulpcore.plugin.models import Content, Repository
+
+populate_content_ids = importlib.import_module(
+ "pulpcore.app.migrations.0152_alter_repositoryversion_content_ids"
+).populate_content_ids
+
+
+def _run_populate():
+ schema_editor = SimpleNamespace(
+ connection=connection,
+ quote_name=connection.ops.quote_name,
+ )
+ populate_content_ids(apps, schema_editor)
+
+
+def _membership_ids(version):
+ return set(version._content_relationships().values_list("content_id", flat=True))
+
+
+@pytest.fixture
+def repository(db):
+ repository = Repository.objects.create(name=uuid4())
+ repository.CONTENT_TYPES = [Content]
+ return repository
+
+
+def test_populate_content_ids_from_content_relationships(repository):
+ contents = [Content(pulp_type="core.content") for _ in range(4)]
+ Content.objects.bulk_create(contents)
+ pks = [c.pk for c in contents]
+
+ version0 = repository.latest_version()
+ with repository.new_version() as version1:
+ version1.add_content(Content.objects.filter(pk__in=pks[:3]))
+ with repository.new_version() as version2:
+ version2.remove_content(Content.objects.filter(pk__in=pks[:1]))
+ with repository.new_version() as version3:
+ version3.add_content(Content.objects.filter(pk__in=pks[3:]))
+
+ # A second repository whose cache is already populated must be left alone.
+ other = Repository.objects.create(name=uuid4())
+ other.CONTENT_TYPES = [Content]
+ with other.new_version() as other_v1:
+ other_v1.add_content(Content.objects.filter(pk__in=pks[:2]))
+ other_v1.refresh_from_db()
+ other_ids_before = list(other_v1.content_ids)
+
+ rv_table = connection.ops.quote_name(RepositoryVersion._meta.db_table)
+ with connection.cursor() as cursor:
+ # Flush deferred triggers from the version inserts so ALTER TABLE can run
+ # inside the test transaction.
+ cursor.execute("SET CONSTRAINTS ALL IMMEDIATE")
+ cursor.execute(f"ALTER TABLE {rv_table} ALTER COLUMN content_ids DROP NOT NULL")
+ # Leave version3 populated (the post-3.83 case) and null the older versions.
+ cursor.execute(
+ f"UPDATE {rv_table} SET content_ids = NULL WHERE repository_id = %s AND number < %s",
+ [repository.pk, version3.number],
+ )
+
+ _run_populate()
+
+ for version in (version0, version1, version2, version3):
+ version.refresh_from_db()
+ assert version.content_ids is not None
+ assert set(version.content_ids) == _membership_ids(version)
+
+ other_v1.refresh_from_db()
+ assert list(other_v1.content_ids) == other_ids_before

CHANGES/+content-ids-migration.bugfix

@@ -0,0 +1 @@
+Filled missing repository version `content_ids` in SQL from content membership, committing per repository so large databases are less likely to hit statement timeouts.

pulpcore/app/migrations/0152_alter_repositoryversion_content_ids.py

@@ -1,35 +1,65 @@
 # Generated by Django 5.2.13 on 2026-06-08 21:54
 
 import django.contrib.postgres.fields
-from django.db import migrations, models
+from django.db import migrations, models, transaction
+
+# Same membership as RepositoryVersion._content_relationships().
+_POPULATE_SQL = """
+UPDATE __RV__ AS rv
+SET content_ids = COALESCE((
+ SELECT ARRAY_AGG(rc.content_id)
+ FROM __RC__ AS rc
+ INNER JOIN __RV__ AS va ON va.pulp_id = rc.version_added_id
+ LEFT JOIN __RV__ AS vr ON vr.pulp_id = rc.version_removed_id
+ WHERE rc.repository_id = rv.repository_id
+ AND va.number <= rv.number
+ AND (vr.pulp_id IS NULL OR vr.number > rv.number)
+), '{}'::uuid[])
+WHERE rv.repository_id = %s
+ AND rv.content_ids IS NULL
+"""
 
 
 def populate_content_ids(apps, schema_editor):
- RepositoryVersion = apps.get_model('core', 'RepositoryVersion')
- RepositoryContent = apps.get_model('core', 'RepositoryContent')
- repo_versions = []
- for rv in RepositoryVersion.objects.filter(content_ids=None).iterator(chunk_size=2000):
- rv.content_ids = list(RepositoryContent.objects.filter(
- repository_id=rv.repository_id, version_added__number__lte=rv.number
- ).exclude(version_removed__number__lte=rv.number).values_list("content_id", flat=True))
- repo_versions.append(rv)
- if len(repo_versions) >= 2000:
- RepositoryVersion.objects.bulk_update(repo_versions, ['content_ids'])
- repo_versions = []
- if repo_versions:
- RepositoryVersion.objects.bulk_update(repo_versions, ['content_ids'])
+ """Fill missing content_ids from content membership, committing per repository."""
+ RepositoryVersion = apps.get_model("core", "RepositoryVersion")
+ RepositoryContent = apps.get_model("core", "RepositoryContent")
+ connection = schema_editor.connection
+ qn = schema_editor.quote_name
+ rv = qn(RepositoryVersion._meta.db_table)
+ rc = qn(RepositoryContent._meta.db_table)
+ sql = _POPULATE_SQL.replace("__RV__", rv).replace("__RC__", rc)
+
+ with connection.cursor() as cursor:
+ cursor.execute(f"SELECT DISTINCT repository_id FROM {rv} WHERE content_ids IS NULL")
+ repo_ids = [row[0] for row in cursor.fetchall()]
+
+ for repo_id in repo_ids:
+ with transaction.atomic(using=connection.alias):
+ with connection.cursor() as cursor:
+ cursor.execute(sql, [repo_id])
+
 
 class Migration(migrations.Migration):
+ # Per-repository commits inside RunPython; AlterField runs afterwards.
+ atomic = False
 
  dependencies = [
- ('core', '0151_upstreampulp_connect_timeout_and_more'),
+ ("core", "0151_upstreampulp_connect_timeout_and_more"),
  ]
 
  operations = [
- migrations.RunPython(populate_content_ids, reverse_code=migrations.RunPython.noop, elidable=True),
+ migrations.RunPython(
+ populate_content_ids,
+ reverse_code=migrations.RunPython.noop,
+ elidable=True,
+ atomic=False,
+ ),
  migrations.AlterField(
- model_name='repositoryversion',
- name='content_ids',
- field=django.contrib.postgres.fields.ArrayField(base_field=models.UUIDField(), default=list, size=None),
+ model_name="repositoryversion",
+ name="content_ids",
+ field=django.contrib.postgres.fields.ArrayField(
+ base_field=models.UUIDField(), default=list, size=None
+ ),
  ),
  ]

pulpcore/tests/unit/models/test_content_ids_migration.py

@@ -0,0 +1,77 @@
+import importlib
+from types import SimpleNamespace
+from uuid import uuid4
+
+import pytest
+from django.apps import apps
+from django.db import connection
+
+from pulpcore.app.models import RepositoryVersion
+from pulpcore.plugin.models import Content, Repository
+
+populate_content_ids = importlib.import_module(
+ "pulpcore.app.migrations.0152_alter_repositoryversion_content_ids"
+).populate_content_ids
+
+
+def _run_populate():
+ schema_editor = SimpleNamespace(
+ connection=connection,
+ quote_name=connection.ops.quote_name,
+ )
+ populate_content_ids(apps, schema_editor)
+
+
+def _membership_ids(version):
+ return set(version._content_relationships().values_list("content_id", flat=True))
+
+
+@pytest.fixture
+def repository(db):
+ repository = Repository.objects.create(name=uuid4())
+ repository.CONTENT_TYPES = [Content]
+ return repository
+
+
+def test_populate_content_ids_from_content_relationships(repository):
+ contents = [Content(pulp_type="core.content") for _ in range(4)]
+ Content.objects.bulk_create(contents)
+ pks = [c.pk for c in contents]
+
+ version0 = repository.latest_version()
+ with repository.new_version() as version1:
+ version1.add_content(Content.objects.filter(pk__in=pks[:3]))
+ with repository.new_version() as version2:
+ version2.remove_content(Content.objects.filter(pk__in=pks[:1]))
+ with repository.new_version() as version3:
+ version3.add_content(Content.objects.filter(pk__in=pks[3:]))
+
+ # A second repository whose cache is already populated must be left alone.
+ other = Repository.objects.create(name=uuid4())
+ other.CONTENT_TYPES = [Content]
+ with other.new_version() as other_v1:
+ other_v1.add_content(Content.objects.filter(pk__in=pks[:2]))
+ other_v1.refresh_from_db()
+ other_ids_before = list(other_v1.content_ids)
+
+ rv_table = connection.ops.quote_name(RepositoryVersion._meta.db_table)
+ with connection.cursor() as cursor:
+ # Flush deferred triggers from the version inserts so ALTER TABLE can run
+ # inside the test transaction.
+ cursor.execute("SET CONSTRAINTS ALL IMMEDIATE")
+ cursor.execute(f"ALTER TABLE {rv_table} ALTER COLUMN content_ids DROP NOT NULL")
+ # Leave version3 populated (the post-3.83 case) and null the older versions.
+ cursor.execute(
+ f"UPDATE {rv_table} SET content_ids = NULL WHERE repository_id = %s AND number < %s",
+ [repository.pk, version3.number],
+ )
+
+ _run_populate()
+
+ for version in (version0, version1, version2, version3):
+ version.refresh_from_db()
+ assert version.content_ids is not None
+ assert set(version.content_ids) == _membership_ids(version)
+
+ other_v1.refresh_from_db()
+ assert list(other_v1.content_ids) == other_ids_before

CHANGES/+content-ids-migration.bugfix

@@ -0,0 +1 @@
+Filled missing repository version `content_ids` in SQL from content membership, committing per repository so large databases are less likely to hit statement timeouts.

pulpcore/app/migrations/0152_alter_repositoryversion_content_ids.py

@@ -1,35 +1,65 @@
 # Generated by Django 5.2.13 on 2026-06-08 21:54
 
 import django.contrib.postgres.fields
-from django.db import migrations, models
+from django.db import migrations, models, transaction
+
+# Same membership as RepositoryVersion._content_relationships().
+_POPULATE_SQL = """
+UPDATE __RV__ AS rv
+SET content_ids = COALESCE((
+ SELECT ARRAY_AGG(rc.content_id)
+ FROM __RC__ AS rc
+ INNER JOIN __RV__ AS va ON va.pulp_id = rc.version_added_id
+ LEFT JOIN __RV__ AS vr ON vr.pulp_id = rc.version_removed_id
+ WHERE rc.repository_id = rv.repository_id
+ AND va.number <= rv.number
+ AND (vr.pulp_id IS NULL OR vr.number > rv.number)
+), '{}'::uuid[])
+WHERE rv.repository_id = %s
+ AND rv.content_ids IS NULL
+"""
 
 
 def populate_content_ids(apps, schema_editor):
- RepositoryVersion = apps.get_model('core', 'RepositoryVersion')
- RepositoryContent = apps.get_model('core', 'RepositoryContent')
- repo_versions = []
- for rv in RepositoryVersion.objects.filter(content_ids=None).iterator(chunk_size=2000):
- rv.content_ids = list(RepositoryContent.objects.filter(
- repository_id=rv.repository_id, version_added__number__lte=rv.number
- ).exclude(version_removed__number__lte=rv.number).values_list("content_id", flat=True))
- repo_versions.append(rv)
- if len(repo_versions) >= 2000:
- RepositoryVersion.objects.bulk_update(repo_versions, ['content_ids'])
- repo_versions = []
- if repo_versions:
- RepositoryVersion.objects.bulk_update(repo_versions, ['content_ids'])
+ """Fill missing content_ids from content membership, committing per repository."""
+ RepositoryVersion = apps.get_model("core", "RepositoryVersion")
+ RepositoryContent = apps.get_model("core", "RepositoryContent")
+ connection = schema_editor.connection
+ qn = schema_editor.quote_name
+ rv = qn(RepositoryVersion._meta.db_table)
+ rc = qn(RepositoryContent._meta.db_table)
+ sql = _POPULATE_SQL.replace("__RV__", rv).replace("__RC__", rc)
+
+ with connection.cursor() as cursor:
+ cursor.execute(f"SELECT DISTINCT repository_id FROM {rv} WHERE content_ids IS NULL")
+ repo_ids = [row[0] for row in cursor.fetchall()]
+
+ for repo_id in repo_ids:
+ with transaction.atomic(using=connection.alias):
+ with connection.cursor() as cursor:
+ cursor.execute(sql, [repo_id])
+
 
 class Migration(migrations.Migration):
+ # Per-repository commits inside RunPython; AlterField runs afterwards.
+ atomic = False
 
  dependencies = [
- ('core', '0151_upstreampulp_connect_timeout_and_more'),
+ ("core", "0151_upstreampulp_connect_timeout_and_more"),
  ]
 
  operations = [
- migrations.RunPython(populate_content_ids, reverse_code=migrations.RunPython.noop, elidable=True),
+ migrations.RunPython(
+ populate_content_ids,
+ reverse_code=migrations.RunPython.noop,
+ elidable=True,
+ atomic=False,
+ ),
  migrations.AlterField(
- model_name='repositoryversion',
- name='content_ids',
- field=django.contrib.postgres.fields.ArrayField(base_field=models.UUIDField(), default=list, size=None),
+ model_name="repositoryversion",
+ name="content_ids",
+ field=django.contrib.postgres.fields.ArrayField(
+ base_field=models.UUIDField(), default=list, size=None
+ ),
  ),
  ]

pulpcore/tests/unit/models/test_content_ids_migration.py

@@ -0,0 +1,77 @@
+import importlib
+from types import SimpleNamespace
+from uuid import uuid4
+
+import pytest
+from django.apps import apps
+from django.db import connection
+
+from pulpcore.app.models import RepositoryVersion
+from pulpcore.plugin.models import Content, Repository
+
+populate_content_ids = importlib.import_module(
+ "pulpcore.app.migrations.0152_alter_repositoryversion_content_ids"
+).populate_content_ids
+
+
+def _run_populate():
+ schema_editor = SimpleNamespace(
+ connection=connection,
+ quote_name=connection.ops.quote_name,
+ )
+ populate_content_ids(apps, schema_editor)
+
+
+def _membership_ids(version):
+ return set(version._content_relationships().values_list("content_id", flat=True))
+
+
+@pytest.fixture
+def repository(db):
+ repository = Repository.objects.create(name=uuid4())
+ repository.CONTENT_TYPES = [Content]
+ return repository
+
+
+def test_populate_content_ids_from_content_relationships(repository):
+ contents = [Content(pulp_type="core.content") for _ in range(4)]
+ Content.objects.bulk_create(contents)
+ pks = [c.pk for c in contents]
+
+ version0 = repository.latest_version()
+ with repository.new_version() as version1:
+ version1.add_content(Content.objects.filter(pk__in=pks[:3]))
+ with repository.new_version() as version2:
+ version2.remove_content(Content.objects.filter(pk__in=pks[:1]))
+ with repository.new_version() as version3:
+ version3.add_content(Content.objects.filter(pk__in=pks[3:]))
+
+ # A second repository whose cache is already populated must be left alone.
+ other = Repository.objects.create(name=uuid4())
+ other.CONTENT_TYPES = [Content]
+ with other.new_version() as other_v1:
+ other_v1.add_content(Content.objects.filter(pk__in=pks[:2]))
+ other_v1.refresh_from_db()
+ other_ids_before = list(other_v1.content_ids)
+
+ rv_table = connection.ops.quote_name(RepositoryVersion._meta.db_table)
+ with connection.cursor() as cursor:
+ # Flush deferred triggers from the version inserts so ALTER TABLE can run
+ # inside the test transaction.
+ cursor.execute("SET CONSTRAINTS ALL IMMEDIATE")
+ cursor.execute(f"ALTER TABLE {rv_table} ALTER COLUMN content_ids DROP NOT NULL")
+ # Leave version3 populated (the post-3.83 case) and null the older versions.
+ cursor.execute(
+ f"UPDATE {rv_table} SET content_ids = NULL WHERE repository_id = %s AND number < %s",
+ [repository.pk, version3.number],
+ )
+
+ _run_populate()
+
+ for version in (version0, version1, version2, version3):
+ version.refresh_from_db()
+ assert version.content_ids is not None
+ assert set(version.content_ids) == _membership_ids(version)
+
+ other_v1.refresh_from_db()
+ assert list(other_v1.content_ids) == other_ids_before

CHANGES/+content-ids-migration.bugfix

@@ -0,0 +1 @@
+Filled missing repository version `content_ids` in SQL from content membership, committing per repository so large databases are less likely to hit statement timeouts.

pulpcore/app/migrations/0152_alter_repositoryversion_content_ids.py

@@ -1,35 +1,65 @@
 # Generated by Django 5.2.13 on 2026-06-08 21:54
 
 import django.contrib.postgres.fields
-from django.db import migrations, models
+from django.db import migrations, models, transaction
+
+# Same membership as RepositoryVersion._content_relationships().
+_POPULATE_SQL = """
+UPDATE __RV__ AS rv
+SET content_ids = COALESCE((
+ SELECT ARRAY_AGG(rc.content_id)
+ FROM __RC__ AS rc
+ INNER JOIN __RV__ AS va ON va.pulp_id = rc.version_added_id
+ LEFT JOIN __RV__ AS vr ON vr.pulp_id = rc.version_removed_id
+ WHERE rc.repository_id = rv.repository_id
+ AND va.number <= rv.number
+ AND (vr.pulp_id IS NULL OR vr.number > rv.number)
+), '{}'::uuid[])
+WHERE rv.repository_id = %s
+ AND rv.content_ids IS NULL
+"""
 
 
 def populate_content_ids(apps, schema_editor):
- RepositoryVersion = apps.get_model('core', 'RepositoryVersion')
- RepositoryContent = apps.get_model('core', 'RepositoryContent')
- repo_versions = []
- for rv in RepositoryVersion.objects.filter(content_ids=None).iterator(chunk_size=2000):
- rv.content_ids = list(RepositoryContent.objects.filter(
- repository_id=rv.repository_id, version_added__number__lte=rv.number
- ).exclude(version_removed__number__lte=rv.number).values_list("content_id", flat=True))
- repo_versions.append(rv)
- if len(repo_versions) >= 2000:
- RepositoryVersion.objects.bulk_update(repo_versions, ['content_ids'])
- repo_versions = []
- if repo_versions:
- RepositoryVersion.objects.bulk_update(repo_versions, ['content_ids'])
+ """Fill missing content_ids from content membership, committing per repository."""
+ RepositoryVersion = apps.get_model("core", "RepositoryVersion")
+ RepositoryContent = apps.get_model("core", "RepositoryContent")
+ connection = schema_editor.connection
+ qn = schema_editor.quote_name
+ rv = qn(RepositoryVersion._meta.db_table)
+ rc = qn(RepositoryContent._meta.db_table)
+ sql = _POPULATE_SQL.replace("__RV__", rv).replace("__RC__", rc)
+
+ with connection.cursor() as cursor:
+ cursor.execute(f"SELECT DISTINCT repository_id FROM {rv} WHERE content_ids IS NULL")
+ repo_ids = [row[0] for row in cursor.fetchall()]
+
+ for repo_id in repo_ids:
+ with transaction.atomic(using=connection.alias):
+ with connection.cursor() as cursor:
+ cursor.execute(sql, [repo_id])
+
 
 class Migration(migrations.Migration):
+ # Per-repository commits inside RunPython; AlterField runs afterwards.
+ atomic = False
 
  dependencies = [
- ('core', '0151_upstreampulp_connect_timeout_and_more'),
+ ("core", "0151_upstreampulp_connect_timeout_and_more"),
  ]
 
  operations = [
- migrations.RunPython(populate_content_ids, reverse_code=migrations.RunPython.noop, elidable=True),
+ migrations.RunPython(
+ populate_content_ids,
+ reverse_code=migrations.RunPython.noop,
+ elidable=True,
+ atomic=False,
+ ),
  migrations.AlterField(
- model_name='repositoryversion',
- name='content_ids',
- field=django.contrib.postgres.fields.ArrayField(base_field=models.UUIDField(), default=list, size=None),
+ model_name="repositoryversion",
+ name="content_ids",
+ field=django.contrib.postgres.fields.ArrayField(
+ base_field=models.UUIDField(), default=list, size=None
+ ),
  ),
  ]

pulpcore/tests/unit/models/test_content_ids_migration.py

@@ -0,0 +1,77 @@
+import importlib
+from types import SimpleNamespace
+from uuid import uuid4
+
+import pytest
+from django.apps import apps
+from django.db import connection
+
+from pulpcore.app.models import RepositoryVersion
+from pulpcore.plugin.models import Content, Repository
+
+populate_content_ids = importlib.import_module(
+ "pulpcore.app.migrations.0152_alter_repositoryversion_content_ids"
+).populate_content_ids
+
+
+def _run_populate():
+ schema_editor = SimpleNamespace(
+ connection=connection,
+ quote_name=connection.ops.quote_name,
+ )
+ populate_content_ids(apps, schema_editor)
+
+
+def _membership_ids(version):
+ return set(version._content_relationships().values_list("content_id", flat=True))
+
+
+@pytest.fixture
+def repository(db):
+ repository = Repository.objects.create(name=uuid4())
+ repository.CONTENT_TYPES = [Content]
+ return repository
+
+
+def test_populate_content_ids_from_content_relationships(repository):
+ contents = [Content(pulp_type="core.content") for _ in range(4)]
+ Content.objects.bulk_create(contents)
+ pks = [c.pk for c in contents]
+
+ version0 = repository.latest_version()
+ with repository.new_version() as version1:
+ version1.add_content(Content.objects.filter(pk__in=pks[:3]))
+ with repository.new_version() as version2:
+ version2.remove_content(Content.objects.filter(pk__in=pks[:1]))
+ with repository.new_version() as version3:
+ version3.add_content(Content.objects.filter(pk__in=pks[3:]))
+
+ # A second repository whose cache is already populated must be left alone.
+ other = Repository.objects.create(name=uuid4())
+ other.CONTENT_TYPES = [Content]
+ with other.new_version() as other_v1:
+ other_v1.add_content(Content.objects.filter(pk__in=pks[:2]))
+ other_v1.refresh_from_db()
+ other_ids_before = list(other_v1.content_ids)
+
+ rv_table = connection.ops.quote_name(RepositoryVersion._meta.db_table)
+ with connection.cursor() as cursor:
+ # Flush deferred triggers from the version inserts so ALTER TABLE can run
+ # inside the test transaction.
+ cursor.execute("SET CONSTRAINTS ALL IMMEDIATE")
+ cursor.execute(f"ALTER TABLE {rv_table} ALTER COLUMN content_ids DROP NOT NULL")
+ # Leave version3 populated (the post-3.83 case) and null the older versions.
+ cursor.execute(
+ f"UPDATE {rv_table} SET content_ids = NULL WHERE repository_id = %s AND number < %s",
+ [repository.pk, version3.number],
+ )
+
+ _run_populate()
+
+ for version in (version0, version1, version2, version3):
+ version.refresh_from_db()
+ assert version.content_ids is not None
+ assert set(version.content_ids) == _membership_ids(version)
+
+ other_v1.refresh_from_db()
+ assert list(other_v1.content_ids) == other_ids_before

CHANGES/+content-ids-migration.bugfix

@@ -0,0 +1 @@
+Filled missing repository version `content_ids` in SQL from content membership, committing per repository so large databases are less likely to hit statement timeouts.

pulpcore/app/migrations/0152_alter_repositoryversion_content_ids.py

@@ -1,35 +1,65 @@
 # Generated by Django 5.2.13 on 2026-06-08 21:54
 
 import django.contrib.postgres.fields
-from django.db import migrations, models
+from django.db import migrations, models, transaction
+
+# Same membership as RepositoryVersion._content_relationships().
+_POPULATE_SQL = """
+UPDATE __RV__ AS rv
+SET content_ids = COALESCE((
+ SELECT ARRAY_AGG(rc.content_id)
+ FROM __RC__ AS rc
+ INNER JOIN __RV__ AS va ON va.pulp_id = rc.version_added_id
+ LEFT JOIN __RV__ AS vr ON vr.pulp_id = rc.version_removed_id
+ WHERE rc.repository_id = rv.repository_id
+ AND va.number <= rv.number
+ AND (vr.pulp_id IS NULL OR vr.number > rv.number)
+), '{}'::uuid[])
+WHERE rv.repository_id = %s
+ AND rv.content_ids IS NULL
+"""
 
 
 def populate_content_ids(apps, schema_editor):
- RepositoryVersion = apps.get_model('core', 'RepositoryVersion')
- RepositoryContent = apps.get_model('core', 'RepositoryContent')
- repo_versions = []
- for rv in RepositoryVersion.objects.filter(content_ids=None).iterator(chunk_size=2000):
- rv.content_ids = list(RepositoryContent.objects.filter(
- repository_id=rv.repository_id, version_added__number__lte=rv.number
- ).exclude(version_removed__number__lte=rv.number).values_list("content_id", flat=True))
- repo_versions.append(rv)
- if len(repo_versions) >= 2000:
- RepositoryVersion.objects.bulk_update(repo_versions, ['content_ids'])
- repo_versions = []
- if repo_versions:
- RepositoryVersion.objects.bulk_update(repo_versions, ['content_ids'])
+ """Fill missing content_ids from content membership, committing per repository."""
+ RepositoryVersion = apps.get_model("core", "RepositoryVersion")
+ RepositoryContent = apps.get_model("core", "RepositoryContent")
+ connection = schema_editor.connection
+ qn = schema_editor.quote_name
+ rv = qn(RepositoryVersion._meta.db_table)
+ rc = qn(RepositoryContent._meta.db_table)
+ sql = _POPULATE_SQL.replace("__RV__", rv).replace("__RC__", rc)
+
+ with connection.cursor() as cursor:
+ cursor.execute(f"SELECT DISTINCT repository_id FROM {rv} WHERE content_ids IS NULL")
+ repo_ids = [row[0] for row in cursor.fetchall()]
+
+ for repo_id in repo_ids:
+ with transaction.atomic(using=connection.alias):
+ with connection.cursor() as cursor:
+ cursor.execute(sql, [repo_id])
+
 
 class Migration(migrations.Migration):
+ # Per-repository commits inside RunPython; AlterField runs afterwards.
+ atomic = False
 
  dependencies = [
- ('core', '0151_upstreampulp_connect_timeout_and_more'),
+ ("core", "0151_upstreampulp_connect_timeout_and_more"),
  ]
 
  operations = [
- migrations.RunPython(populate_content_ids, reverse_code=migrations.RunPython.noop, elidable=True),
+ migrations.RunPython(
+ populate_content_ids,
+ reverse_code=migrations.RunPython.noop,
+ elidable=True,
+ atomic=False,
+ ),
  migrations.AlterField(
- model_name='repositoryversion',
- name='content_ids',
- field=django.contrib.postgres.fields.ArrayField(base_field=models.UUIDField(), default=list, size=None),
+ model_name="repositoryversion",
+ name="content_ids",
+ field=django.contrib.postgres.fields.ArrayField(
+ base_field=models.UUIDField(), default=list, size=None
+ ),
  ),
  ]

pulpcore/tests/unit/models/test_content_ids_migration.py

@@ -0,0 +1,77 @@
+import importlib
+from types import SimpleNamespace
+from uuid import uuid4
+
+import pytest
+from django.apps import apps
+from django.db import connection
+
+from pulpcore.app.models import RepositoryVersion
+from pulpcore.plugin.models import Content, Repository
+
+populate_content_ids = importlib.import_module(
+ "pulpcore.app.migrations.0152_alter_repositoryversion_content_ids"
+).populate_content_ids
+
+
+def _run_populate():
+ schema_editor = SimpleNamespace(
+ connection=connection,
+ quote_name=connection.ops.quote_name,
+ )
+ populate_content_ids(apps, schema_editor)
+
+
+def _membership_ids(version):
+ return set(version._content_relationships().values_list("content_id", flat=True))
+
+
+@pytest.fixture
+def repository(db):
+ repository = Repository.objects.create(name=uuid4())
+ repository.CONTENT_TYPES = [Content]
+ return repository
+
+
+def test_populate_content_ids_from_content_relationships(repository):
+ contents = [Content(pulp_type="core.content") for _ in range(4)]
+ Content.objects.bulk_create(contents)
+ pks = [c.pk for c in contents]
+
+ version0 = repository.latest_version()
+ with repository.new_version() as version1:
+ version1.add_content(Content.objects.filter(pk__in=pks[:3]))
+ with repository.new_version() as version2:
+ version2.remove_content(Content.objects.filter(pk__in=pks[:1]))
+ with repository.new_version() as version3:
+ version3.add_content(Content.objects.filter(pk__in=pks[3:]))
+
+ # A second repository whose cache is already populated must be left alone.
+ other = Repository.objects.create(name=uuid4())
+ other.CONTENT_TYPES = [Content]
+ with other.new_version() as other_v1:
+ other_v1.add_content(Content.objects.filter(pk__in=pks[:2]))
+ other_v1.refresh_from_db()
+ other_ids_before = list(other_v1.content_ids)
+
+ rv_table = connection.ops.quote_name(RepositoryVersion._meta.db_table)
+ with connection.cursor() as cursor:
+ # Flush deferred triggers from the version inserts so ALTER TABLE can run
+ # inside the test transaction.
+ cursor.execute("SET CONSTRAINTS ALL IMMEDIATE")
+ cursor.execute(f"ALTER TABLE {rv_table} ALTER COLUMN content_ids DROP NOT NULL")
+ # Leave version3 populated (the post-3.83 case) and null the older versions.
+ cursor.execute(
+ f"UPDATE {rv_table} SET content_ids = NULL WHERE repository_id = %s AND number < %s",
+ [repository.pk, version3.number],
+ )
+
+ _run_populate()
+
+ for version in (version0, version1, version2, version3):
+ version.refresh_from_db()
+ assert version.content_ids is not None
+ assert set(version.content_ids) == _membership_ids(version)
+
+ other_v1.refresh_from_db()
+ assert list(other_v1.content_ids) == other_ids_before

CHANGES/+content-ids-migration.bugfix

@@ -0,0 +1 @@
+Filled missing repository version `content_ids` in SQL from content membership, committing per repository so large databases are less likely to hit statement timeouts.

pulpcore/app/migrations/0152_alter_repositoryversion_content_ids.py

@@ -1,35 +1,65 @@
 # Generated by Django 5.2.13 on 2026-06-08 21:54
 
 import django.contrib.postgres.fields
-from django.db import migrations, models
+from django.db import migrations, models, transaction
+
+# Same membership as RepositoryVersion._content_relationships().
+_POPULATE_SQL = """
+UPDATE __RV__ AS rv
+SET content_ids = COALESCE((
+ SELECT ARRAY_AGG(rc.content_id)
+ FROM __RC__ AS rc
+ INNER JOIN __RV__ AS va ON va.pulp_id = rc.version_added_id
+ LEFT JOIN __RV__ AS vr ON vr.pulp_id = rc.version_removed_id
+ WHERE rc.repository_id = rv.repository_id
+ AND va.number <= rv.number
+ AND (vr.pulp_id IS NULL OR vr.number > rv.number)
+), '{}'::uuid[])
+WHERE rv.repository_id = %s
+ AND rv.content_ids IS NULL
+"""
 
 
 def populate_content_ids(apps, schema_editor):
- RepositoryVersion = apps.get_model('core', 'RepositoryVersion')
- RepositoryContent = apps.get_model('core', 'RepositoryContent')
- repo_versions = []
- for rv in RepositoryVersion.objects.filter(content_ids=None).iterator(chunk_size=2000):
- rv.content_ids = list(RepositoryContent.objects.filter(
- repository_id=rv.repository_id, version_added__number__lte=rv.number
- ).exclude(version_removed__number__lte=rv.number).values_list("content_id", flat=True))
- repo_versions.append(rv)
- if len(repo_versions) >= 2000:
- RepositoryVersion.objects.bulk_update(repo_versions, ['content_ids'])
- repo_versions = []
- if repo_versions:
- RepositoryVersion.objects.bulk_update(repo_versions, ['content_ids'])
+ """Fill missing content_ids from content membership, committing per repository."""
+ RepositoryVersion = apps.get_model("core", "RepositoryVersion")
+ RepositoryContent = apps.get_model("core", "RepositoryContent")
+ connection = schema_editor.connection
+ qn = schema_editor.quote_name
+ rv = qn(RepositoryVersion._meta.db_table)
+ rc = qn(RepositoryContent._meta.db_table)
+ sql = _POPULATE_SQL.replace("__RV__", rv).replace("__RC__", rc)
+
+ with connection.cursor() as cursor:
+ cursor.execute(f"SELECT DISTINCT repository_id FROM {rv} WHERE content_ids IS NULL")
+ repo_ids = [row[0] for row in cursor.fetchall()]
+
+ for repo_id in repo_ids:
+ with transaction.atomic(using=connection.alias):
+ with connection.cursor() as cursor:
+ cursor.execute(sql, [repo_id])
+
 
 class Migration(migrations.Migration):
+ # Per-repository commits inside RunPython; AlterField runs afterwards.
+ atomic = False
 
  dependencies = [
- ('core', '0151_upstreampulp_connect_timeout_and_more'),
+ ("core", "0151_upstreampulp_connect_timeout_and_more"),
  ]
 
  operations = [
- migrations.RunPython(populate_content_ids, reverse_code=migrations.RunPython.noop, elidable=True),
+ migrations.RunPython(
+ populate_content_ids,
+ reverse_code=migrations.RunPython.noop,
+ elidable=True,
+ atomic=False,
+ ),
  migrations.AlterField(
- model_name='repositoryversion',
- name='content_ids',
- field=django.contrib.postgres.fields.ArrayField(base_field=models.UUIDField(), default=list, size=None),
+ model_name="repositoryversion",
+ name="content_ids",
+ field=django.contrib.postgres.fields.ArrayField(
+ base_field=models.UUIDField(), default=list, size=None
+ ),
  ),
  ]

pulpcore/tests/unit/models/test_content_ids_migration.py

@@ -0,0 +1,77 @@
+import importlib
+from types import SimpleNamespace
+from uuid import uuid4
+
+import pytest
+from django.apps import apps
+from django.db import connection
+
+from pulpcore.app.models import RepositoryVersion
+from pulpcore.plugin.models import Content, Repository
+
+populate_content_ids = importlib.import_module(
+ "pulpcore.app.migrations.0152_alter_repositoryversion_content_ids"
+).populate_content_ids
+
+
+def _run_populate():
+ schema_editor = SimpleNamespace(
+ connection=connection,
+ quote_name=connection.ops.quote_name,
+ )
+ populate_content_ids(apps, schema_editor)
+
+
+def _membership_ids(version):
+ return set(version._content_relationships().values_list("content_id", flat=True))
+
+
+@pytest.fixture
+def repository(db):
+ repository = Repository.objects.create(name=uuid4())
+ repository.CONTENT_TYPES = [Content]
+ return repository
+
+
+def test_populate_content_ids_from_content_relationships(repository):
+ contents = [Content(pulp_type="core.content") for _ in range(4)]
+ Content.objects.bulk_create(contents)
+ pks = [c.pk for c in contents]
+
+ version0 = repository.latest_version()
+ with repository.new_version() as version1:
+ version1.add_content(Content.objects.filter(pk__in=pks[:3]))
+ with repository.new_version() as version2:
+ version2.remove_content(Content.objects.filter(pk__in=pks[:1]))
+ with repository.new_version() as version3:
+ version3.add_content(Content.objects.filter(pk__in=pks[3:]))
+
+ # A second repository whose cache is already populated must be left alone.
+ other = Repository.objects.create(name=uuid4())
+ other.CONTENT_TYPES = [Content]
+ with other.new_version() as other_v1:
+ other_v1.add_content(Content.objects.filter(pk__in=pks[:2]))
+ other_v1.refresh_from_db()
+ other_ids_before = list(other_v1.content_ids)
+
+ rv_table = connection.ops.quote_name(RepositoryVersion._meta.db_table)
+ with connection.cursor() as cursor:
+ # Flush deferred triggers from the version inserts so ALTER TABLE can run
+ # inside the test transaction.
+ cursor.execute("SET CONSTRAINTS ALL IMMEDIATE")
+ cursor.execute(f"ALTER TABLE {rv_table} ALTER COLUMN content_ids DROP NOT NULL")
+ # Leave version3 populated (the post-3.83 case) and null the older versions.
+ cursor.execute(
+ f"UPDATE {rv_table} SET content_ids = NULL WHERE repository_id = %s AND number < %s",
+ [repository.pk, version3.number],
+ )
+
+ _run_populate()
+
+ for version in (version0, version1, version2, version3):
+ version.refresh_from_db()
+ assert version.content_ids is not None
+ assert set(version.content_ids) == _membership_ids(version)
+
+ other_v1.refresh_from_db()
+ assert list(other_v1.content_ids) == other_ids_before

CHANGES/+content-ids-migration.bugfix

@@ -0,0 +1 @@
+Filled missing repository version `content_ids` in SQL from content membership, committing per repository so large databases are less likely to hit statement timeouts.

pulpcore/app/migrations/0152_alter_repositoryversion_content_ids.py

@@ -1,35 +1,65 @@
 # Generated by Django 5.2.13 on 2026-06-08 21:54
 
 import django.contrib.postgres.fields
-from django.db import migrations, models
+from django.db import migrations, models, transaction
+
+# Same membership as RepositoryVersion._content_relationships().
+_POPULATE_SQL = """
+UPDATE __RV__ AS rv
+SET content_ids = COALESCE((
+ SELECT ARRAY_AGG(rc.content_id)
+ FROM __RC__ AS rc
+ INNER JOIN __RV__ AS va ON va.pulp_id = rc.version_added_id
+ LEFT JOIN __RV__ AS vr ON vr.pulp_id = rc.version_removed_id
+ WHERE rc.repository_id = rv.repository_id
+ AND va.number <= rv.number
+ AND (vr.pulp_id IS NULL OR vr.number > rv.number)
+), '{}'::uuid[])
+WHERE rv.repository_id = %s
+ AND rv.content_ids IS NULL
+"""
 
 
 def populate_content_ids(apps, schema_editor):
- RepositoryVersion = apps.get_model('core', 'RepositoryVersion')
- RepositoryContent = apps.get_model('core', 'RepositoryContent')
- repo_versions = []
- for rv in RepositoryVersion.objects.filter(content_ids=None).iterator(chunk_size=2000):
- rv.content_ids = list(RepositoryContent.objects.filter(
- repository_id=rv.repository_id, version_added__number__lte=rv.number
- ).exclude(version_removed__number__lte=rv.number).values_list("content_id", flat=True))
- repo_versions.append(rv)
- if len(repo_versions) >= 2000:
- RepositoryVersion.objects.bulk_update(repo_versions, ['content_ids'])
- repo_versions = []
- if repo_versions:
- RepositoryVersion.objects.bulk_update(repo_versions, ['content_ids'])
+ """Fill missing content_ids from content membership, committing per repository."""
+ RepositoryVersion = apps.get_model("core", "RepositoryVersion")
+ RepositoryContent = apps.get_model("core", "RepositoryContent")
+ connection = schema_editor.connection
+ qn = schema_editor.quote_name
+ rv = qn(RepositoryVersion._meta.db_table)
+ rc = qn(RepositoryContent._meta.db_table)
+ sql = _POPULATE_SQL.replace("__RV__", rv).replace("__RC__", rc)
+
+ with connection.cursor() as cursor:
+ cursor.execute(f"SELECT DISTINCT repository_id FROM {rv} WHERE content_ids IS NULL")
+ repo_ids = [row[0] for row in cursor.fetchall()]
+
+ for repo_id in repo_ids:
+ with transaction.atomic(using=connection.alias):
+ with connection.cursor() as cursor:
+ cursor.execute(sql, [repo_id])
+
 
 class Migration(migrations.Migration):
+ # Per-repository commits inside RunPython; AlterField runs afterwards.
+ atomic = False
 
  dependencies = [
- ('core', '0151_upstreampulp_connect_timeout_and_more'),
+ ("core", "0151_upstreampulp_connect_timeout_and_more"),
  ]
 
  operations = [
- migrations.RunPython(populate_content_ids, reverse_code=migrations.RunPython.noop, elidable=True),
+ migrations.RunPython(
+ populate_content_ids,
+ reverse_code=migrations.RunPython.noop,
+ elidable=True,
+ atomic=False,
+ ),
  migrations.AlterField(
- model_name='repositoryversion',
- name='content_ids',
- field=django.contrib.postgres.fields.ArrayField(base_field=models.UUIDField(), default=list, size=None),
+ model_name="repositoryversion",
+ name="content_ids",
+ field=django.contrib.postgres.fields.ArrayField(
+ base_field=models.UUIDField(), default=list, size=None
+ ),
  ),
  ]

pulpcore/tests/unit/models/test_content_ids_migration.py

@@ -0,0 +1,77 @@
+import importlib
+from types import SimpleNamespace
+from uuid import uuid4
+
+import pytest
+from django.apps import apps
+from django.db import connection
+
+from pulpcore.app.models import RepositoryVersion
+from pulpcore.plugin.models import Content, Repository
+
+populate_content_ids = importlib.import_module(
+ "pulpcore.app.migrations.0152_alter_repositoryversion_content_ids"
+).populate_content_ids
+
+
+def _run_populate():
+ schema_editor = SimpleNamespace(
+ connection=connection,
+ quote_name=connection.ops.quote_name,
+ )
+ populate_content_ids(apps, schema_editor)
+
+
+def _membership_ids(version):
+ return set(version._content_relationships().values_list("content_id", flat=True))
+
+
+@pytest.fixture
+def repository(db):
+ repository = Repository.objects.create(name=uuid4())
+ repository.CONTENT_TYPES = [Content]
+ return repository
+
+
+def test_populate_content_ids_from_content_relationships(repository):
+ contents = [Content(pulp_type="core.content") for _ in range(4)]
+ Content.objects.bulk_create(contents)
+ pks = [c.pk for c in contents]
+
+ version0 = repository.latest_version()
+ with repository.new_version() as version1:
+ version1.add_content(Content.objects.filter(pk__in=pks[:3]))
+ with repository.new_version() as version2:
+ version2.remove_content(Content.objects.filter(pk__in=pks[:1]))
+ with repository.new_version() as version3:
+ version3.add_content(Content.objects.filter(pk__in=pks[3:]))
+
+ # A second repository whose cache is already populated must be left alone.
+ other = Repository.objects.create(name=uuid4())
+ other.CONTENT_TYPES = [Content]
+ with other.new_version() as other_v1:
+ other_v1.add_content(Content.objects.filter(pk__in=pks[:2]))
+ other_v1.refresh_from_db()
+ other_ids_before = list(other_v1.content_ids)
+
+ rv_table = connection.ops.quote_name(RepositoryVersion._meta.db_table)
+ with connection.cursor() as cursor:
+ # Flush deferred triggers from the version inserts so ALTER TABLE can run
+ # inside the test transaction.
+ cursor.execute("SET CONSTRAINTS ALL IMMEDIATE")
+ cursor.execute(f"ALTER TABLE {rv_table} ALTER COLUMN content_ids DROP NOT NULL")
+ # Leave version3 populated (the post-3.83 case) and null the older versions.
+ cursor.execute(
+ f"UPDATE {rv_table} SET content_ids = NULL WHERE repository_id = %s AND number < %s",
+ [repository.pk, version3.number],
+ )
+
+ _run_populate()
+
+ for version in (version0, version1, version2, version3):
+ version.refresh_from_db()
+ assert version.content_ids is not None
+ assert set(version.content_ids) == _membership_ids(version)
+
+ other_v1.refresh_from_db()
+ assert list(other_v1.content_ids) == other_ids_before

CHANGES/+content-ids-migration.bugfix

@@ -0,0 +1 @@
+Filled missing repository version `content_ids` in SQL from content membership, committing per repository so large databases are less likely to hit statement timeouts.

pulpcore/app/migrations/0152_alter_repositoryversion_content_ids.py

@@ -1,35 +1,65 @@
 # Generated by Django 5.2.13 on 2026-06-08 21:54
 
 import django.contrib.postgres.fields
-from django.db import migrations, models
+from django.db import migrations, models, transaction
+
+# Same membership as RepositoryVersion._content_relationships().
+_POPULATE_SQL = """
+UPDATE __RV__ AS rv
+SET content_ids = COALESCE((
+ SELECT ARRAY_AGG(rc.content_id)
+ FROM __RC__ AS rc
+ INNER JOIN __RV__ AS va ON va.pulp_id = rc.version_added_id
+ LEFT JOIN __RV__ AS vr ON vr.pulp_id = rc.version_removed_id
+ WHERE rc.repository_id = rv.repository_id
+ AND va.number <= rv.number
+ AND (vr.pulp_id IS NULL OR vr.number > rv.number)
+), '{}'::uuid[])
+WHERE rv.repository_id = %s
+ AND rv.content_ids IS NULL
+"""
 
 
 def populate_content_ids(apps, schema_editor):
- RepositoryVersion = apps.get_model('core', 'RepositoryVersion')
- RepositoryContent = apps.get_model('core', 'RepositoryContent')
- repo_versions = []
- for rv in RepositoryVersion.objects.filter(content_ids=None).iterator(chunk_size=2000):
- rv.content_ids = list(RepositoryContent.objects.filter(
- repository_id=rv.repository_id, version_added__number__lte=rv.number
- ).exclude(version_removed__number__lte=rv.number).values_list("content_id", flat=True))
- repo_versions.append(rv)
- if len(repo_versions) >= 2000:
- RepositoryVersion.objects.bulk_update(repo_versions, ['content_ids'])
- repo_versions = []
- if repo_versions:
- RepositoryVersion.objects.bulk_update(repo_versions, ['content_ids'])
+ """Fill missing content_ids from content membership, committing per repository."""
+ RepositoryVersion = apps.get_model("core", "RepositoryVersion")
+ RepositoryContent = apps.get_model("core", "RepositoryContent")
+ connection = schema_editor.connection
+ qn = schema_editor.quote_name
+ rv = qn(RepositoryVersion._meta.db_table)
+ rc = qn(RepositoryContent._meta.db_table)
+ sql = _POPULATE_SQL.replace("__RV__", rv).replace("__RC__", rc)
+
+ with connection.cursor() as cursor:
+ cursor.execute(f"SELECT DISTINCT repository_id FROM {rv} WHERE content_ids IS NULL")
+ repo_ids = [row[0] for row in cursor.fetchall()]
+
+ for repo_id in repo_ids:
+ with transaction.atomic(using=connection.alias):
+ with connection.cursor() as cursor:
+ cursor.execute(sql, [repo_id])
+
 
 class Migration(migrations.Migration):
+ # Per-repository commits inside RunPython; AlterField runs afterwards.
+ atomic = False
 
  dependencies = [
- ('core', '0151_upstreampulp_connect_timeout_and_more'),
+ ("core", "0151_upstreampulp_connect_timeout_and_more"),
  ]
 
  operations = [
- migrations.RunPython(populate_content_ids, reverse_code=migrations.RunPython.noop, elidable=True),
+ migrations.RunPython(
+ populate_content_ids,
+ reverse_code=migrations.RunPython.noop,
+ elidable=True,
+ atomic=False,
+ ),
  migrations.AlterField(
- model_name='repositoryversion',
- name='content_ids',
- field=django.contrib.postgres.fields.ArrayField(base_field=models.UUIDField(), default=list, size=None),
+ model_name="repositoryversion",
+ name="content_ids",
+ field=django.contrib.postgres.fields.ArrayField(
+ base_field=models.UUIDField(), default=list, size=None
+ ),
  ),
  ]

pulpcore/tests/unit/models/test_content_ids_migration.py

@@ -0,0 +1,77 @@
+import importlib
+from types import SimpleNamespace
+from uuid import uuid4
+
+import pytest
+from django.apps import apps
+from django.db import connection
+
+from pulpcore.app.models import RepositoryVersion
+from pulpcore.plugin.models import Content, Repository
+
+populate_content_ids = importlib.import_module(
+ "pulpcore.app.migrations.0152_alter_repositoryversion_content_ids"
+).populate_content_ids
+
+
+def _run_populate():
+ schema_editor = SimpleNamespace(
+ connection=connection,
+ quote_name=connection.ops.quote_name,
+ )
+ populate_content_ids(apps, schema_editor)
+
+
+def _membership_ids(version):
+ return set(version._content_relationships().values_list("content_id", flat=True))
+
+
+@pytest.fixture
+def repository(db):
+ repository = Repository.objects.create(name=uuid4())
+ repository.CONTENT_TYPES = [Content]
+ return repository
+
+
+def test_populate_content_ids_from_content_relationships(repository):
+ contents = [Content(pulp_type="core.content") for _ in range(4)]
+ Content.objects.bulk_create(contents)
+ pks = [c.pk for c in contents]
+
+ version0 = repository.latest_version()
+ with repository.new_version() as version1:
+ version1.add_content(Content.objects.filter(pk__in=pks[:3]))
+ with repository.new_version() as version2:
+ version2.remove_content(Content.objects.filter(pk__in=pks[:1]))
+ with repository.new_version() as version3:
+ version3.add_content(Content.objects.filter(pk__in=pks[3:]))
+
+ # A second repository whose cache is already populated must be left alone.
+ other = Repository.objects.create(name=uuid4())
+ other.CONTENT_TYPES = [Content]
+ with other.new_version() as other_v1:
+ other_v1.add_content(Content.objects.filter(pk__in=pks[:2]))
+ other_v1.refresh_from_db()
+ other_ids_before = list(other_v1.content_ids)
+
+ rv_table = connection.ops.quote_name(RepositoryVersion._meta.db_table)
+ with connection.cursor() as cursor:
+ # Flush deferred triggers from the version inserts so ALTER TABLE can run
+ # inside the test transaction.
+ cursor.execute("SET CONSTRAINTS ALL IMMEDIATE")
+ cursor.execute(f"ALTER TABLE {rv_table} ALTER COLUMN content_ids DROP NOT NULL")
+ # Leave version3 populated (the post-3.83 case) and null the older versions.
+ cursor.execute(
+ f"UPDATE {rv_table} SET content_ids = NULL WHERE repository_id = %s AND number < %s",
+ [repository.pk, version3.number],
+ )
+
+ _run_populate()
+
+ for version in (version0, version1, version2, version3):
+ version.refresh_from_db()
+ assert version.content_ids is not None
+ assert set(version.content_ids) == _membership_ids(version)
+
+ other_v1.refresh_from_db()
+ assert list(other_v1.content_ids) == other_ids_before