Added logs to duplicate_assessment_items command

contentcuration/contentcuration/management/commands/fix_duplicate_assessment_items.py

@@ -2,8 +2,8 @@
 import time
 import uuid
 
+import progressbar
 from django.core.management.base import BaseCommand
-from django.db import transaction
 from django.db.models import Count
 from django.db.models import F
 
@@ -18,42 +18,47 @@ class Command(BaseCommand):
  def handle(self, *args, **options):
  start = time.time()
  # Go through nodes that have assessment items with the same assessment_id
+ logging.info("Looking for nodes with invalid assessments...")
  nodes = ContentNode.objects.filter(kind_id='exercise') \
  .annotate(
  num_ids=Count('assessment_items__pk'),
  num_assessment_ids=Count('assessment_items__assessment_id', distinct=True)
  ).exclude(num_ids=F('num_assessment_ids'))
+ total = nodes.count()
 
- with transaction.atomic():
- items_to_delete = []
- for node in nodes:
- # Go through each node's assessment items
- for item in node.assessment_items.all():
- # Handle duplicate assessment ids
- exclude_ids = [i.pk for i in items_to_delete] + [item.pk]
- other_duplicate_assessment_items = node.assessment_items.filter(assessment_id=item.assessment_id).exclude(pk__in=exclude_ids)
-
- if other_duplicate_assessment_items.exists():
- # Remove duplicates
- if other_duplicate_assessment_items.filter(
- question=item.question,
- answers=item.answers,
- hints=item.hints,
- raw_data=item.raw_data
- ).exists():
- items_to_delete.append(item)
-
- # Get new ids for non-duplicates
- else:
+ logging.info("Fixing {} nodes...".format(total))
+
+ items_to_delete = []
+ bar = progressbar.ProgressBar(max_value=total)
+ for i, node in enumerate(nodes):
+ # Go through each node's assessment items
+ for item in node.assessment_items.all():
+ # Handle duplicate assessment ids
+ exclude_ids = [i.pk for i in items_to_delete] + [item.pk]
+ other_duplicate_assessment_items = node.assessment_items.filter(assessment_id=item.assessment_id).exclude(pk__in=exclude_ids)
+
+ if other_duplicate_assessment_items.exists():
+ # Remove duplicates
+ if other_duplicate_assessment_items.filter(
+ question=item.question,
+ answers=item.answers,
+ hints=item.hints,
+ raw_data=item.raw_data
+ ).exists():
+ items_to_delete.append(item)
+
+ # Get new ids for non-duplicates
+ else:
+ new_id = uuid.uuid4().hex
+ while node.assessment_items.filter(assessment_id=new_id).exists():
  new_id = uuid.uuid4().hex
- while node.assessment_items.filter(assessment_id=new_id).exists():
- new_id = uuid.uuid4().hex
- item.assessment_id = new_id
- item.save()
+ item.assessment_id = new_id
+ item.save()
+ bar.update(i)
 
- for item in items_to_delete:
- if AssessmentItem.objects.filter(assessment_id=item.assessment_id, contentnode_id=item.contentnode_id).exclude(pk=item.pk).exists():
- item.delete()
+ logging.info("Deduping identical assessments...")
+ for item in items_to_delete:
+ if AssessmentItem.objects.filter(assessment_id=item.assessment_id, contentnode_id=item.contentnode_id).exclude(pk=item.pk).exists():
+ item.delete()
 
- self.stdout.write("Finished in {}".format(time.time() - start))
  logging.info("Finished in {}".format(time.time() - start))

contentcuration/contentcuration/migrations/0130_merge_20210113_1839.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2021-01-13 18:39
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('contentcuration', '0129_auto_20210113_1723'),
+ ('contentcuration', '0129_merge_20210112_2327'),
+ ]
+
+ operations = [
+ ]

contentcuration/contentcuration/management/commands/fix_duplicate_assessment_items.py

@@ -2,8 +2,8 @@
 import time
 import uuid
 
+import progressbar
 from django.core.management.base import BaseCommand
-from django.db import transaction
 from django.db.models import Count
 from django.db.models import F
 
@@ -18,42 +18,47 @@ class Command(BaseCommand):
  def handle(self, *args, **options):
  start = time.time()
  # Go through nodes that have assessment items with the same assessment_id
+ logging.info("Looking for nodes with invalid assessments...")
  nodes = ContentNode.objects.filter(kind_id='exercise') \
  .annotate(
  num_ids=Count('assessment_items__pk'),
  num_assessment_ids=Count('assessment_items__assessment_id', distinct=True)
  ).exclude(num_ids=F('num_assessment_ids'))
+ total = nodes.count()
 
- with transaction.atomic():
- items_to_delete = []
- for node in nodes:
- # Go through each node's assessment items
- for item in node.assessment_items.all():
- # Handle duplicate assessment ids
- exclude_ids = [i.pk for i in items_to_delete] + [item.pk]
- other_duplicate_assessment_items = node.assessment_items.filter(assessment_id=item.assessment_id).exclude(pk__in=exclude_ids)
-
- if other_duplicate_assessment_items.exists():
- # Remove duplicates
- if other_duplicate_assessment_items.filter(
- question=item.question,
- answers=item.answers,
- hints=item.hints,
- raw_data=item.raw_data
- ).exists():
- items_to_delete.append(item)
-
- # Get new ids for non-duplicates
- else:
+ logging.info("Fixing {} nodes...".format(total))
+
+ items_to_delete = []
+ bar = progressbar.ProgressBar(max_value=total)
+ for i, node in enumerate(nodes):
+ # Go through each node's assessment items
+ for item in node.assessment_items.all():
+ # Handle duplicate assessment ids
+ exclude_ids = [i.pk for i in items_to_delete] + [item.pk]
+ other_duplicate_assessment_items = node.assessment_items.filter(assessment_id=item.assessment_id).exclude(pk__in=exclude_ids)
+
+ if other_duplicate_assessment_items.exists():
+ # Remove duplicates
+ if other_duplicate_assessment_items.filter(
+ question=item.question,
+ answers=item.answers,
+ hints=item.hints,
+ raw_data=item.raw_data
+ ).exists():
+ items_to_delete.append(item)
+
+ # Get new ids for non-duplicates
+ else:
+ new_id = uuid.uuid4().hex
+ while node.assessment_items.filter(assessment_id=new_id).exists():
  new_id = uuid.uuid4().hex
- while node.assessment_items.filter(assessment_id=new_id).exists():
- new_id = uuid.uuid4().hex
- item.assessment_id = new_id
- item.save()
+ item.assessment_id = new_id
+ item.save()
+ bar.update(i)
 
- for item in items_to_delete:
- if AssessmentItem.objects.filter(assessment_id=item.assessment_id, contentnode_id=item.contentnode_id).exclude(pk=item.pk).exists():
- item.delete()
+ logging.info("Deduping identical assessments...")
+ for item in items_to_delete:
+ if AssessmentItem.objects.filter(assessment_id=item.assessment_id, contentnode_id=item.contentnode_id).exclude(pk=item.pk).exists():
+ item.delete()
 
- self.stdout.write("Finished in {}".format(time.time() - start))
  logging.info("Finished in {}".format(time.time() - start))

contentcuration/contentcuration/migrations/0130_merge_20210113_1839.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2021-01-13 18:39
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('contentcuration', '0129_auto_20210113_1723'),
+ ('contentcuration', '0129_merge_20210112_2327'),
+ ]
+
+ operations = [
+ ]

contentcuration/contentcuration/management/commands/fix_duplicate_assessment_items.py

@@ -2,8 +2,8 @@
 import time
 import uuid
 
+import progressbar
 from django.core.management.base import BaseCommand
-from django.db import transaction
 from django.db.models import Count
 from django.db.models import F
 
@@ -18,42 +18,47 @@ class Command(BaseCommand):
  def handle(self, *args, **options):
  start = time.time()
  # Go through nodes that have assessment items with the same assessment_id
+ logging.info("Looking for nodes with invalid assessments...")
  nodes = ContentNode.objects.filter(kind_id='exercise') \
  .annotate(
  num_ids=Count('assessment_items__pk'),
  num_assessment_ids=Count('assessment_items__assessment_id', distinct=True)
  ).exclude(num_ids=F('num_assessment_ids'))
+ total = nodes.count()
 
- with transaction.atomic():
- items_to_delete = []
- for node in nodes:
- # Go through each node's assessment items
- for item in node.assessment_items.all():
- # Handle duplicate assessment ids
- exclude_ids = [i.pk for i in items_to_delete] + [item.pk]
- other_duplicate_assessment_items = node.assessment_items.filter(assessment_id=item.assessment_id).exclude(pk__in=exclude_ids)
-
- if other_duplicate_assessment_items.exists():
- # Remove duplicates
- if other_duplicate_assessment_items.filter(
- question=item.question,
- answers=item.answers,
- hints=item.hints,
- raw_data=item.raw_data
- ).exists():
- items_to_delete.append(item)
-
- # Get new ids for non-duplicates
- else:
+ logging.info("Fixing {} nodes...".format(total))
+
+ items_to_delete = []
+ bar = progressbar.ProgressBar(max_value=total)
+ for i, node in enumerate(nodes):
+ # Go through each node's assessment items
+ for item in node.assessment_items.all():
+ # Handle duplicate assessment ids
+ exclude_ids = [i.pk for i in items_to_delete] + [item.pk]
+ other_duplicate_assessment_items = node.assessment_items.filter(assessment_id=item.assessment_id).exclude(pk__in=exclude_ids)
+
+ if other_duplicate_assessment_items.exists():
+ # Remove duplicates
+ if other_duplicate_assessment_items.filter(
+ question=item.question,
+ answers=item.answers,
+ hints=item.hints,
+ raw_data=item.raw_data
+ ).exists():
+ items_to_delete.append(item)
+
+ # Get new ids for non-duplicates
+ else:
+ new_id = uuid.uuid4().hex
+ while node.assessment_items.filter(assessment_id=new_id).exists():
  new_id = uuid.uuid4().hex
- while node.assessment_items.filter(assessment_id=new_id).exists():
- new_id = uuid.uuid4().hex
- item.assessment_id = new_id
- item.save()
+ item.assessment_id = new_id
+ item.save()
+ bar.update(i)
 
- for item in items_to_delete:
- if AssessmentItem.objects.filter(assessment_id=item.assessment_id, contentnode_id=item.contentnode_id).exclude(pk=item.pk).exists():
- item.delete()
+ logging.info("Deduping identical assessments...")
+ for item in items_to_delete:
+ if AssessmentItem.objects.filter(assessment_id=item.assessment_id, contentnode_id=item.contentnode_id).exclude(pk=item.pk).exists():
+ item.delete()
 
- self.stdout.write("Finished in {}".format(time.time() - start))
  logging.info("Finished in {}".format(time.time() - start))

contentcuration/contentcuration/migrations/0130_merge_20210113_1839.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2021-01-13 18:39
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('contentcuration', '0129_auto_20210113_1723'),
+ ('contentcuration', '0129_merge_20210112_2327'),
+ ]
+
+ operations = [
+ ]

contentcuration/contentcuration/management/commands/fix_duplicate_assessment_items.py

@@ -2,8 +2,8 @@
 import time
 import uuid
 
+import progressbar
 from django.core.management.base import BaseCommand
-from django.db import transaction
 from django.db.models import Count
 from django.db.models import F
 
@@ -18,42 +18,47 @@ class Command(BaseCommand):
  def handle(self, *args, **options):
  start = time.time()
  # Go through nodes that have assessment items with the same assessment_id
+ logging.info("Looking for nodes with invalid assessments...")
  nodes = ContentNode.objects.filter(kind_id='exercise') \
  .annotate(
  num_ids=Count('assessment_items__pk'),
  num_assessment_ids=Count('assessment_items__assessment_id', distinct=True)
  ).exclude(num_ids=F('num_assessment_ids'))
+ total = nodes.count()
 
- with transaction.atomic():
- items_to_delete = []
- for node in nodes:
- # Go through each node's assessment items
- for item in node.assessment_items.all():
- # Handle duplicate assessment ids
- exclude_ids = [i.pk for i in items_to_delete] + [item.pk]
- other_duplicate_assessment_items = node.assessment_items.filter(assessment_id=item.assessment_id).exclude(pk__in=exclude_ids)
-
- if other_duplicate_assessment_items.exists():
- # Remove duplicates
- if other_duplicate_assessment_items.filter(
- question=item.question,
- answers=item.answers,
- hints=item.hints,
- raw_data=item.raw_data
- ).exists():
- items_to_delete.append(item)
-
- # Get new ids for non-duplicates
- else:
+ logging.info("Fixing {} nodes...".format(total))
+
+ items_to_delete = []
+ bar = progressbar.ProgressBar(max_value=total)
+ for i, node in enumerate(nodes):
+ # Go through each node's assessment items
+ for item in node.assessment_items.all():
+ # Handle duplicate assessment ids
+ exclude_ids = [i.pk for i in items_to_delete] + [item.pk]
+ other_duplicate_assessment_items = node.assessment_items.filter(assessment_id=item.assessment_id).exclude(pk__in=exclude_ids)
+
+ if other_duplicate_assessment_items.exists():
+ # Remove duplicates
+ if other_duplicate_assessment_items.filter(
+ question=item.question,
+ answers=item.answers,
+ hints=item.hints,
+ raw_data=item.raw_data
+ ).exists():
+ items_to_delete.append(item)
+
+ # Get new ids for non-duplicates
+ else:
+ new_id = uuid.uuid4().hex
+ while node.assessment_items.filter(assessment_id=new_id).exists():
  new_id = uuid.uuid4().hex
- while node.assessment_items.filter(assessment_id=new_id).exists():
- new_id = uuid.uuid4().hex
- item.assessment_id = new_id
- item.save()
+ item.assessment_id = new_id
+ item.save()
+ bar.update(i)
 
- for item in items_to_delete:
- if AssessmentItem.objects.filter(assessment_id=item.assessment_id, contentnode_id=item.contentnode_id).exclude(pk=item.pk).exists():
- item.delete()
+ logging.info("Deduping identical assessments...")
+ for item in items_to_delete:
+ if AssessmentItem.objects.filter(assessment_id=item.assessment_id, contentnode_id=item.contentnode_id).exclude(pk=item.pk).exists():
+ item.delete()
 
- self.stdout.write("Finished in {}".format(time.time() - start))
  logging.info("Finished in {}".format(time.time() - start))

contentcuration/contentcuration/migrations/0130_merge_20210113_1839.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2021-01-13 18:39
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('contentcuration', '0129_auto_20210113_1723'),
+ ('contentcuration', '0129_merge_20210112_2327'),
+ ]
+
+ operations = [
+ ]

contentcuration/contentcuration/management/commands/fix_duplicate_assessment_items.py

@@ -2,8 +2,8 @@
 import time
 import uuid
 
+import progressbar
 from django.core.management.base import BaseCommand
-from django.db import transaction
 from django.db.models import Count
 from django.db.models import F
 
@@ -18,42 +18,47 @@ class Command(BaseCommand):
  def handle(self, *args, **options):
  start = time.time()
  # Go through nodes that have assessment items with the same assessment_id
+ logging.info("Looking for nodes with invalid assessments...")
  nodes = ContentNode.objects.filter(kind_id='exercise') \
  .annotate(
  num_ids=Count('assessment_items__pk'),
  num_assessment_ids=Count('assessment_items__assessment_id', distinct=True)
  ).exclude(num_ids=F('num_assessment_ids'))
+ total = nodes.count()
 
- with transaction.atomic():
- items_to_delete = []
- for node in nodes:
- # Go through each node's assessment items
- for item in node.assessment_items.all():
- # Handle duplicate assessment ids
- exclude_ids = [i.pk for i in items_to_delete] + [item.pk]
- other_duplicate_assessment_items = node.assessment_items.filter(assessment_id=item.assessment_id).exclude(pk__in=exclude_ids)
-
- if other_duplicate_assessment_items.exists():
- # Remove duplicates
- if other_duplicate_assessment_items.filter(
- question=item.question,
- answers=item.answers,
- hints=item.hints,
- raw_data=item.raw_data
- ).exists():
- items_to_delete.append(item)
-
- # Get new ids for non-duplicates
- else:
+ logging.info("Fixing {} nodes...".format(total))
+
+ items_to_delete = []
+ bar = progressbar.ProgressBar(max_value=total)
+ for i, node in enumerate(nodes):
+ # Go through each node's assessment items
+ for item in node.assessment_items.all():
+ # Handle duplicate assessment ids
+ exclude_ids = [i.pk for i in items_to_delete] + [item.pk]
+ other_duplicate_assessment_items = node.assessment_items.filter(assessment_id=item.assessment_id).exclude(pk__in=exclude_ids)
+
+ if other_duplicate_assessment_items.exists():
+ # Remove duplicates
+ if other_duplicate_assessment_items.filter(
+ question=item.question,
+ answers=item.answers,
+ hints=item.hints,
+ raw_data=item.raw_data
+ ).exists():
+ items_to_delete.append(item)
+
+ # Get new ids for non-duplicates
+ else:
+ new_id = uuid.uuid4().hex
+ while node.assessment_items.filter(assessment_id=new_id).exists():
  new_id = uuid.uuid4().hex
- while node.assessment_items.filter(assessment_id=new_id).exists():
- new_id = uuid.uuid4().hex
- item.assessment_id = new_id
- item.save()
+ item.assessment_id = new_id
+ item.save()
+ bar.update(i)
 
- for item in items_to_delete:
- if AssessmentItem.objects.filter(assessment_id=item.assessment_id, contentnode_id=item.contentnode_id).exclude(pk=item.pk).exists():
- item.delete()
+ logging.info("Deduping identical assessments...")
+ for item in items_to_delete:
+ if AssessmentItem.objects.filter(assessment_id=item.assessment_id, contentnode_id=item.contentnode_id).exclude(pk=item.pk).exists():
+ item.delete()
 
- self.stdout.write("Finished in {}".format(time.time() - start))
  logging.info("Finished in {}".format(time.time() - start))

contentcuration/contentcuration/migrations/0130_merge_20210113_1839.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2021-01-13 18:39
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('contentcuration', '0129_auto_20210113_1723'),
+ ('contentcuration', '0129_merge_20210112_2327'),
+ ]
+
+ operations = [
+ ]

contentcuration/contentcuration/management/commands/fix_duplicate_assessment_items.py

@@ -2,8 +2,8 @@
 import time
 import uuid
 
+import progressbar
 from django.core.management.base import BaseCommand
-from django.db import transaction
 from django.db.models import Count
 from django.db.models import F
 
@@ -18,42 +18,47 @@ class Command(BaseCommand):
  def handle(self, *args, **options):
  start = time.time()
  # Go through nodes that have assessment items with the same assessment_id
+ logging.info("Looking for nodes with invalid assessments...")
  nodes = ContentNode.objects.filter(kind_id='exercise') \
  .annotate(
  num_ids=Count('assessment_items__pk'),
  num_assessment_ids=Count('assessment_items__assessment_id', distinct=True)
  ).exclude(num_ids=F('num_assessment_ids'))
+ total = nodes.count()
 
- with transaction.atomic():
- items_to_delete = []
- for node in nodes:
- # Go through each node's assessment items
- for item in node.assessment_items.all():
- # Handle duplicate assessment ids
- exclude_ids = [i.pk for i in items_to_delete] + [item.pk]
- other_duplicate_assessment_items = node.assessment_items.filter(assessment_id=item.assessment_id).exclude(pk__in=exclude_ids)
-
- if other_duplicate_assessment_items.exists():
- # Remove duplicates
- if other_duplicate_assessment_items.filter(
- question=item.question,
- answers=item.answers,
- hints=item.hints,
- raw_data=item.raw_data
- ).exists():
- items_to_delete.append(item)
-
- # Get new ids for non-duplicates
- else:
+ logging.info("Fixing {} nodes...".format(total))
+
+ items_to_delete = []
+ bar = progressbar.ProgressBar(max_value=total)
+ for i, node in enumerate(nodes):
+ # Go through each node's assessment items
+ for item in node.assessment_items.all():
+ # Handle duplicate assessment ids
+ exclude_ids = [i.pk for i in items_to_delete] + [item.pk]
+ other_duplicate_assessment_items = node.assessment_items.filter(assessment_id=item.assessment_id).exclude(pk__in=exclude_ids)
+
+ if other_duplicate_assessment_items.exists():
+ # Remove duplicates
+ if other_duplicate_assessment_items.filter(
+ question=item.question,
+ answers=item.answers,
+ hints=item.hints,
+ raw_data=item.raw_data
+ ).exists():
+ items_to_delete.append(item)
+
+ # Get new ids for non-duplicates
+ else:
+ new_id = uuid.uuid4().hex
+ while node.assessment_items.filter(assessment_id=new_id).exists():
  new_id = uuid.uuid4().hex
- while node.assessment_items.filter(assessment_id=new_id).exists():
- new_id = uuid.uuid4().hex
- item.assessment_id = new_id
- item.save()
+ item.assessment_id = new_id
+ item.save()
+ bar.update(i)
 
- for item in items_to_delete:
- if AssessmentItem.objects.filter(assessment_id=item.assessment_id, contentnode_id=item.contentnode_id).exclude(pk=item.pk).exists():
- item.delete()
+ logging.info("Deduping identical assessments...")
+ for item in items_to_delete:
+ if AssessmentItem.objects.filter(assessment_id=item.assessment_id, contentnode_id=item.contentnode_id).exclude(pk=item.pk).exists():
+ item.delete()
 
- self.stdout.write("Finished in {}".format(time.time() - start))
  logging.info("Finished in {}".format(time.time() - start))

contentcuration/contentcuration/migrations/0130_merge_20210113_1839.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2021-01-13 18:39
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('contentcuration', '0129_auto_20210113_1723'),
+ ('contentcuration', '0129_merge_20210112_2327'),
+ ]
+
+ operations = [
+ ]

contentcuration/contentcuration/management/commands/fix_duplicate_assessment_items.py

@@ -2,8 +2,8 @@
 import time
 import uuid
 
+import progressbar
 from django.core.management.base import BaseCommand
-from django.db import transaction
 from django.db.models import Count
 from django.db.models import F
 
@@ -18,42 +18,47 @@ class Command(BaseCommand):
  def handle(self, *args, **options):
  start = time.time()
  # Go through nodes that have assessment items with the same assessment_id
+ logging.info("Looking for nodes with invalid assessments...")
  nodes = ContentNode.objects.filter(kind_id='exercise') \
  .annotate(
  num_ids=Count('assessment_items__pk'),
  num_assessment_ids=Count('assessment_items__assessment_id', distinct=True)
  ).exclude(num_ids=F('num_assessment_ids'))
+ total = nodes.count()
 
- with transaction.atomic():
- items_to_delete = []
- for node in nodes:
- # Go through each node's assessment items
- for item in node.assessment_items.all():
- # Handle duplicate assessment ids
- exclude_ids = [i.pk for i in items_to_delete] + [item.pk]
- other_duplicate_assessment_items = node.assessment_items.filter(assessment_id=item.assessment_id).exclude(pk__in=exclude_ids)
-
- if other_duplicate_assessment_items.exists():
- # Remove duplicates
- if other_duplicate_assessment_items.filter(
- question=item.question,
- answers=item.answers,
- hints=item.hints,
- raw_data=item.raw_data
- ).exists():
- items_to_delete.append(item)
-
- # Get new ids for non-duplicates
- else:
+ logging.info("Fixing {} nodes...".format(total))
+
+ items_to_delete = []
+ bar = progressbar.ProgressBar(max_value=total)
+ for i, node in enumerate(nodes):
+ # Go through each node's assessment items
+ for item in node.assessment_items.all():
+ # Handle duplicate assessment ids
+ exclude_ids = [i.pk for i in items_to_delete] + [item.pk]
+ other_duplicate_assessment_items = node.assessment_items.filter(assessment_id=item.assessment_id).exclude(pk__in=exclude_ids)
+
+ if other_duplicate_assessment_items.exists():
+ # Remove duplicates
+ if other_duplicate_assessment_items.filter(
+ question=item.question,
+ answers=item.answers,
+ hints=item.hints,
+ raw_data=item.raw_data
+ ).exists():
+ items_to_delete.append(item)
+
+ # Get new ids for non-duplicates
+ else:
+ new_id = uuid.uuid4().hex
+ while node.assessment_items.filter(assessment_id=new_id).exists():
  new_id = uuid.uuid4().hex
- while node.assessment_items.filter(assessment_id=new_id).exists():
- new_id = uuid.uuid4().hex
- item.assessment_id = new_id
- item.save()
+ item.assessment_id = new_id
+ item.save()
+ bar.update(i)
 
- for item in items_to_delete:
- if AssessmentItem.objects.filter(assessment_id=item.assessment_id, contentnode_id=item.contentnode_id).exclude(pk=item.pk).exists():
- item.delete()
+ logging.info("Deduping identical assessments...")
+ for item in items_to_delete:
+ if AssessmentItem.objects.filter(assessment_id=item.assessment_id, contentnode_id=item.contentnode_id).exclude(pk=item.pk).exists():
+ item.delete()
 
- self.stdout.write("Finished in {}".format(time.time() - start))
  logging.info("Finished in {}".format(time.time() - start))

contentcuration/contentcuration/migrations/0130_merge_20210113_1839.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2021-01-13 18:39
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('contentcuration', '0129_auto_20210113_1723'),
+ ('contentcuration', '0129_merge_20210112_2327'),
+ ]
+
+ operations = [
+ ]

contentcuration/contentcuration/management/commands/fix_duplicate_assessment_items.py

@@ -2,8 +2,8 @@
 import time
 import uuid
 
+import progressbar
 from django.core.management.base import BaseCommand
-from django.db import transaction
 from django.db.models import Count
 from django.db.models import F
 
@@ -18,42 +18,47 @@ class Command(BaseCommand):
  def handle(self, *args, **options):
  start = time.time()
  # Go through nodes that have assessment items with the same assessment_id
+ logging.info("Looking for nodes with invalid assessments...")
  nodes = ContentNode.objects.filter(kind_id='exercise') \
  .annotate(
  num_ids=Count('assessment_items__pk'),
  num_assessment_ids=Count('assessment_items__assessment_id', distinct=True)
  ).exclude(num_ids=F('num_assessment_ids'))
+ total = nodes.count()
 
- with transaction.atomic():
- items_to_delete = []
- for node in nodes:
- # Go through each node's assessment items
- for item in node.assessment_items.all():
- # Handle duplicate assessment ids
- exclude_ids = [i.pk for i in items_to_delete] + [item.pk]
- other_duplicate_assessment_items = node.assessment_items.filter(assessment_id=item.assessment_id).exclude(pk__in=exclude_ids)
-
- if other_duplicate_assessment_items.exists():
- # Remove duplicates
- if other_duplicate_assessment_items.filter(
- question=item.question,
- answers=item.answers,
- hints=item.hints,
- raw_data=item.raw_data
- ).exists():
- items_to_delete.append(item)
-
- # Get new ids for non-duplicates
- else:
+ logging.info("Fixing {} nodes...".format(total))
+
+ items_to_delete = []
+ bar = progressbar.ProgressBar(max_value=total)
+ for i, node in enumerate(nodes):
+ # Go through each node's assessment items
+ for item in node.assessment_items.all():
+ # Handle duplicate assessment ids
+ exclude_ids = [i.pk for i in items_to_delete] + [item.pk]
+ other_duplicate_assessment_items = node.assessment_items.filter(assessment_id=item.assessment_id).exclude(pk__in=exclude_ids)
+
+ if other_duplicate_assessment_items.exists():
+ # Remove duplicates
+ if other_duplicate_assessment_items.filter(
+ question=item.question,
+ answers=item.answers,
+ hints=item.hints,
+ raw_data=item.raw_data
+ ).exists():
+ items_to_delete.append(item)
+
+ # Get new ids for non-duplicates
+ else:
+ new_id = uuid.uuid4().hex
+ while node.assessment_items.filter(assessment_id=new_id).exists():
  new_id = uuid.uuid4().hex
- while node.assessment_items.filter(assessment_id=new_id).exists():
- new_id = uuid.uuid4().hex
- item.assessment_id = new_id
- item.save()
+ item.assessment_id = new_id
+ item.save()
+ bar.update(i)
 
- for item in items_to_delete:
- if AssessmentItem.objects.filter(assessment_id=item.assessment_id, contentnode_id=item.contentnode_id).exclude(pk=item.pk).exists():
- item.delete()
+ logging.info("Deduping identical assessments...")
+ for item in items_to_delete:
+ if AssessmentItem.objects.filter(assessment_id=item.assessment_id, contentnode_id=item.contentnode_id).exclude(pk=item.pk).exists():
+ item.delete()
 
- self.stdout.write("Finished in {}".format(time.time() - start))
  logging.info("Finished in {}".format(time.time() - start))

contentcuration/contentcuration/migrations/0130_merge_20210113_1839.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2021-01-13 18:39
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('contentcuration', '0129_auto_20210113_1723'),
+ ('contentcuration', '0129_merge_20210112_2327'),
+ ]
+
+ operations = [
+ ]