strip string values imported from C5 tables; force materialized views computation for manually imported batches

Author: beda42

apps/logs/logic/custom_import.py

@@ -8,6 +8,7 @@
 
 from core.models import UL_ORG_ADMIN
 from logs.logic.data_import import import_counter_records
+from logs.logic.materialized_reports import sync_materialized_reports_for_import_batch
 from nigiri.counter5 import CounterRecord
 from logs.models import ImportBatch, ManualDataUpload, Metric, OrganizationPlatform
 
@@ -163,6 +164,7 @@ def import_custom_data(mdu: ManualDataUpload, user) -> dict:
     OrganizationPlatform.objects.get_or_create(platform=mdu.platform, organization=mdu.organization)
     mdu.import_batch = import_batch
     mdu.mark_processed()
+    sync_materialized_reports_for_import_batch(import_batch)
     # the following could be used to debug the speed of this code chunk
     # qs = connection.queries
     # qs.sort(key=lambda x: -float(x['time']))

apps/logs/logic/materialized_reports.py

@@ -22,6 +22,16 @@ def sync_materialized_reports():
         create_materialized_accesslogs(mat_rt)
 
 
+def sync_materialized_reports_for_import_batch(ib: ImportBatch):
+    """
+    Create AccessLogs for all materialized report types for one import batch
+    """
+    for mat_rt in ReportType.objects.filter(
+        materialization_spec__isnull=False, materialization_spec__base_report_type=ib.report_type
+    ):
+        create_materialized_accesslogs_for_importbatches(mat_rt, [ib])
+
+
 def create_materialized_accesslogs(rt: ReportType, batch_size=None) -> int:
     """
     Given an input materialized report type, it creates all the missing accesslogs. It detects

apps/nigiri/counter5.py

@@ -238,10 +238,10 @@ def _fd_to_records(self, infile) -> typing.Generator[CounterRecord, None, None]:
         header = {}
         reader = csv.reader(infile, dialect=dialect)
         for header_line in reader:
-            if not header_line[0]:
+            if not header_line[0].strip():
                 # we break on empty line - it means end of header and start of data
                 break
-            header[header_line[0]] = header_line[1]
+            header[header_line[0].strip()] = header_line[1].strip()
         report_type = header.get('Report_ID')
         if not report_type or report_type not in self.report_type_to_dimensions:
             raise ValueError(f'Unsupported report type: {report_type}')
@@ -258,13 +258,15 @@ def _fd_to_records(self, infile) -> typing.Generator[CounterRecord, None, None]:
             monthly_values = {}
             title_ids = {}
             for key, value in record.items():
-                if not value.strip():
+                key = key.strip()
+                value = value.strip()
+                if not value:
                     continue
                 month = parse_counter_month(key)
                 if month:
                     monthly_values[month] = int(value)
                 else:
-                    if not key.strip() or key in self.ignored_columns:
+                    if not key or key in self.ignored_columns:
                         pass
                     elif key in self.column_map:
                         implicit_dimensions[self.column_map[key]] = value

config/settings/devel.py

@@ -3,7 +3,7 @@
 from .base import *
 
 DATABASES['default']['NAME'] = 'celus'
-DATABASES['default']['PORT'] = 5434
+DATABASES['default']['PORT'] = 5432
 ALLOWED_HOSTS = ['*']
 
 EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'