Add command to reset contract documents (#177)

* Add command to reset contract documents

* Handle missing data from contractors

* Keep track of last processed contract

* Skip doc processing

* Add aux redis cache

* Cache forever

* Handle null exempt_id

Co-authored-by: José Padilla <[email protected]>

Author: jpadilla

contratospr/contracts/admin.py

@@ -68,13 +68,23 @@ class ContractorAdmin(admin.ModelAdmin):
     search_fields = ["name", "source_id"]
 
 
+class ContractInline(admin.StackedInline):
+    model = Contract
+    exclude = ["search_vector"]
+    raw_id_fields = ["entity", "service", "document", "contractors", "parent"]
+    can_delete = False
+    extra = 0
+    max_num = 1
+
+
 @admin.register(Document)
 class DocumentAdmin(admin.ModelAdmin):
     list_display = ["source_id", "file", "has_text", "created_at", "modified_at"]
     exclude = ["pages"]
     search_fields = ["source_id"]
     actions = ["download_source", "detect_text"]
     list_filter = [DocumentFileListFilter]
+    inlines = [ContractInline]
 
     def has_text(self, obj):
         return bool(obj.pages)

contratospr/contracts/management/commands/reset_documents.py

@@ -0,0 +1,26 @@
+from django.core.management.base import BaseCommand
+
+from ...models import Document
+from ...search import index_contract
+
+
+class Command(BaseCommand):
+    help = "Delete all documents in S3, clears extracted pages, and reindex contracts."
+
+    def handle(self, *args, **options):
+        documents = Document.objects.exclude(file="").defer("pages")
+
+        for document in documents:
+            self.stdout.write(f"=> Resetting document {document.pk}")
+
+            self.stdout.write("==> Deleting file")
+            document.file.delete()
+
+            self.stdout.write("==> Clearing pages")
+            document.pages = None
+            document.save(update_fields=["pages"])
+
+            self.stdout.write("==> Indexing contracts")
+            for contract in document.contract_set.all():
+                self.stdout.write(f"===> Indexing contract {contract.pk}")
+                index_contract(contract)

contratospr/contracts/management/commands/update_contracts.py

@@ -0,0 +1,64 @@
+from django.core.cache import caches
+from django.core.management.base import BaseCommand
+
+from ...models import Contract
+from ...tasks import scrape_contracts
+
+
+class Command(BaseCommand):
+    help = "Update local contracts with data from remote source"
+
+    def add_arguments(self, parser):
+        parser.add_argument("--limit", nargs="?", type=int, default=1000)
+
+    def handle(self, *args, **options):
+        try:
+            cache = caches["aux"]
+        except Exception:
+            cache = caches["default"]
+
+        limit = options.get("limit")
+
+        contracts = (
+            Contract.objects.select_related("entity")
+            .all()
+            .order_by("pk")
+            .only("pk", "number", "entity")
+        )
+
+        cur_offset = 0
+        cur_limit = 1
+
+        cache_key = "cmd:update_contracts:last_preview_id:limit={}".format(limit)
+        last_contract_id = cache.get(cache_key)
+
+        if last_contract_id:
+            self.stdout.write("=> Starting after {}".format(last_contract_id))
+            contracts = contracts.filter(pk__gt=last_contract_id)
+
+        while cur_limit <= limit:
+            cur_c = list(contracts[cur_offset:cur_limit].iterator())
+
+            if len(cur_c) > 0:
+                c = cur_c[0]
+                last_contract_id = c.pk
+
+                self.stdout.write(f"=> Scraping contract {c.pk} / {c.number}")
+                scrape_contracts(
+                    skip_doc_tasks=True,
+                    contract_number=c.number,
+                    entity_id=c.entity.source_id,
+                )
+
+                self.stdout.write("=> Last contract {}".format(last_contract_id))
+                cache.set(cache_key, last_contract_id, timeout=None)
+            else:
+                self.stdout.write(
+                    "=> Nothing found offset={} / limit={} / last_contract_id={}".format(
+                        cur_offset, cur_limit, last_contract_id
+                    )
+                )
+                break
+
+            cur_offset += 1
+            cur_limit += 1

contratospr/contracts/migrations/0007_auto_20210912_2302.py

@@ -0,0 +1,16 @@
+# Generated by Django 3.1.12 on 2021-09-12 23:02
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [("contracts", "0006_auto_20201212_2129")]
+
+    operations = [
+        migrations.AlterField(
+            model_name="document",
+            name="source_id",
+            field=models.CharField(max_length=255, unique=True),
+        )
+    ]

contratospr/contracts/migrations/0008_auto_20211218_1611.py

@@ -0,0 +1,16 @@
+# Generated by Django 3.1.12 on 2021-12-18 16:11
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [("contracts", "0007_auto_20210912_2302")]
+
+    operations = [
+        migrations.AlterField(
+            model_name="contract",
+            name="exempt_id",
+            field=models.CharField(blank=True, max_length=255),
+        )
+    ]

contratospr/contracts/models.py

@@ -68,7 +68,7 @@ def __str__(self):
 
 
 class Document(BaseModel):
-    source_id = models.PositiveIntegerField(unique=True)
+    source_id = models.CharField(max_length=255, unique=True)
     source_url = models.URLField()
     file = models.FileField(
         blank=True, null=True, upload_to=document_file_path, storage=document_storage
@@ -126,7 +126,7 @@ class Contract(BaseModel):
     amount_to_pay = models.DecimalField(max_digits=20, decimal_places=2)
     has_amendments = models.BooleanField()
     document = models.ForeignKey("Document", null=True, on_delete=models.SET_NULL)
-    exempt_id = models.CharField(max_length=255)
+    exempt_id = models.CharField(max_length=255, blank=True)
     contractors = models.ManyToManyField("Contractor")
     parent = models.ForeignKey(
         "self", null=True, on_delete=models.CASCADE, related_name="amendments"

contratospr/contracts/tasks.py

@@ -65,7 +65,7 @@ def normalize_contract(contract):
         document_id = result["document_id"]
         result[
             "document_url"
-        ] = f"{BASE_CONTRACT_URL}/downloaddocument?documentid={document_id}"
+        ] = f"{BASE_CONTRACT_URL}/downloaddocument?code={document_id}"
 
     return result
 
@@ -78,7 +78,9 @@ def normalize_contractors(contractors):
             {
                 "contractor_id": contractor["ContractorId"],
                 "entity_id": contractor["EntityId"],
-                "name": contractor["Name"],
+                "name": contractor["Name"]
+                or contractor["ConfirmedName1"]
+                or contractor["ConfirmedName2"],
             }
         )
 
@@ -138,7 +140,7 @@ def request_contract_document(contract_id):
 
 
 @app.task
-def update_contract(result, parent_id=None):
+def update_contract(result, parent_id=None, skip_doc_tasks=False):
     logger.info(
         "Updating contract", contract=result["contract_number"], parent_id=parent_id
     )
@@ -171,7 +173,7 @@ def update_contract(result, parent_id=None):
         "cancellation_date": result["cancellation_date"],
         "amount_to_pay": result["amount_to_pay"],
         "has_amendments": result["has_amendments"],
-        "exempt_id": result["exempt_id"],
+        "exempt_id": result["exempt_id"] or "",
         "parent_id": parent_id,
     }
 
@@ -181,7 +183,7 @@ def update_contract(result, parent_id=None):
             defaults={"source_url": result["document_url"]},
         )
 
-        if document_created:
+        if document_created and skip_doc_tasks:
             chain(download_document.si(document.pk), detect_text.si(document.pk))()
 
         artifacts.append({"obj": document, "created": document_created})
@@ -195,7 +197,7 @@ def update_contract(result, parent_id=None):
 
     for contractor_result in result["contractors"]:
         contractor, contractor_created = Contractor.objects.get_or_create(
-            source_id=contractor_result["contractor_id"],
+            source_id=contractor_result["contractor_id"] or contract.source_id,
             defaults={
                 "name": contractor_result["name"],
                 "entity_id": contractor_result["entity_id"],
@@ -207,10 +209,13 @@ def update_contract(result, parent_id=None):
         contract.contractors.add(contractor)
 
     for amendment_result in result["amendments"]:
-        amendment_artifacts = update_contract(amendment_result, parent_id=contract.pk)
+        amendment_artifacts = update_contract(
+            amendment_result, parent_id=contract.pk, skip_doc_tasks=skip_doc_tasks
+        )
         artifacts.extend(amendment_artifacts)
 
-    index_contract(contract)
+    if not skip_doc_tasks:
+        index_contract(contract)
 
     return artifacts
 
@@ -221,6 +226,7 @@ def scrape_contracts(limit=None, max_items=None, **kwargs):
     total_records = 0
     default_limit = 10
     real_limit = limit or default_limit
+    skip_doc_tasks = kwargs.pop("skip_doc_tasks", False)
     collection_job_id = kwargs.pop("collection_job_id", None)
     collection_job = None
     if collection_job_id:
@@ -242,7 +248,7 @@ def scrape_contracts(limit=None, max_items=None, **kwargs):
 
         for contract in contracts["data"]:
             expanded = expand_contract(contract)
-            results = update_contract(expanded)
+            results = update_contract(expanded, skip_doc_tasks=skip_doc_tasks)
 
             if collection_job:
                 collection_job.create_artifacts(results)

contratospr/settings.py

@@ -151,6 +151,7 @@ class Common(Configuration):
     AUTH_USER_MODEL = "users.User"
 
     REDIS_URL = values.Value(environ_prefix=None)
+    AUX_REDIS_URL = values.Value(environ_prefix=None)
 
     DEBUG_TOOLBAR_CONFIG = {
         "SHOW_TOOLBAR_CALLBACK": "contratospr.utils.debug_toolbar.show_toolbar"
@@ -229,14 +230,23 @@ class Production(Common):
 
     @property
     def CACHES(self):
-        return {
+        caches = {
             "default": {
                 "BACKEND": "django_redis.cache.RedisCache",
                 "LOCATION": f"{self.REDIS_URL}/1",
                 "OPTIONS": {"CLIENT_CLASS": "django_redis.client.DefaultClient"},
             }
         }
 
+        if self.AUX_REDIS_URL:
+            caches["aux"] = {
+                "BACKEND": "django_redis.cache.RedisCache",
+                "LOCATION": f"{self.AUX_REDIS_URL}/1",
+                "OPTIONS": {"CLIENT_CLASS": "django_redis.client.DefaultClient"},
+            }
+
+        return caches
+
 
 class Testing(Common):
     PASSWORD_HASHERS = ["django.contrib.auth.hashers.MD5PasswordHasher"]