From 0c84417d03042064949ec633d2380c716d971fe2 Mon Sep 17 00:00:00 2001 From: ayishanishana21 Date: Thu, 22 Jan 2026 13:11:49 +0530 Subject: [PATCH 1/3] test request view --- cron/tasks.py | 142 ++++++++++++++++++++++++++++- events/views.py | 235 +++++++++++++++++++++++++++++++++++------------- 2 files changed, 316 insertions(+), 61 deletions(-) diff --git a/cron/tasks.py b/cron/tasks.py index c0a0b96f6..68e189b9f 100644 --- a/cron/tasks.py +++ b/cron/tasks.py @@ -30,6 +30,15 @@ from events.models import FossMdlCourses, TestAttendance, State, City, InstituteType from creation.models import FossCategory +from django.db import transaction +from training.models import TrainingAttend +from events.models import Test, TestAttendance +from mdldjango.helper import get_moodle_user +from events.helpers import get_fossmdlcourse +from django.db import close_old_connections + + + def bulk_email(taskid, *args, **kwargs): task = AsyncCronMail.objects.get(pk=taskid) if task.log_file.name == "": @@ -166,4 +175,135 @@ def filter_student_grades(key=None): def async_filter_student_grades(key): - TOPPER_QUEUE.enqueue(filter_student_grades, key, job_id=key, job_timeout='72h') \ No newline at end of file + TOPPER_QUEUE.enqueue(filter_student_grades, key, job_id=key, job_timeout='72h') + + + +def process_test_attendance(test_id): + """ + Background task: + - Create TestAttendance + - Sync Moodle users + """ + close_old_connections() + job = get_current_job() + if job: + job.meta['test_id'] = test_id + job.save_meta() + + try: + test = Test.objects.select_related('training', 'foss').get(pk=test_id) + except Test.DoesNotExist: + return + + if not test.training_id: + return + + tras = TrainingAttend.objects.select_related( + 'student__user', + 'training__training_planner' + ).filter(training=test.training) + + fossmdlcourse = get_fossmdlcourse( + test.foss_id, + fossmdlmap_id=test.training.fossmdlmap_id + ) + + existing = { + (x.student_id, x.mdluser_id) + for x in TestAttendance.objects.filter(test=test) + } + + mdluser_cache = {} + new_rows = [] + + for tra in tras: + user = tra.student.user + + key = ( + tra.training.training_planner.academic_id, + user.first_name, + user.last_name, + tra.student.gender, + user.email + ) + + if key not in mdluser_cache: + mdluser_cache[key] = get_moodle_user(*key) + + mdluser = mdluser_cache[key] + if not mdluser: + continue + + pair = (tra.student.id, mdluser.id) + if pair in existing: + continue + + new_rows.append( + TestAttendance( + student_id=tra.student.id, + test=test, + mdluser_id=mdluser.id, + mdlcourse_id=fossmdlcourse.mdlcourse_id, + mdlquiz_id=fossmdlcourse.mdlquiz_id, + mdlattempt_id=0, + status=0 + ) + ) + + if new_rows: + close_old_connections() + with transaction.atomic(): + TestAttendance.objects.bulk_create(new_rows) + + +def process_test_post_save(test_id, user_id, message,academic_id): + """ + Background task: + - Event log + - Notifications + """ + close_old_connections() + from events.views import ( + update_events_log, + update_events_notification + ) + + update_events_log( + user_id=user_id, + role=0, + category=1, + category_id=test_id, + academic=academic_id, + status=0 + ) + + update_events_notification( + user_id=user_id, + role=0, + category=1, + category_id=test_id, + academic=academic_id, + status=0, + message=message + ) + + +def async_process_test_attendance(test): + DEFAULT_QUEUE.enqueue( + process_test_attendance, + test.pk, + job_id="test_attendance_%s" % test.pk, + job_timeout='72h' + ) + + +def async_test_post_save(test, user, message): + DEFAULT_QUEUE.enqueue( + process_test_post_save, + test.pk, + user.pk, + message, + job_id="test_post_save_%s" % test.pk, + job_timeout='24h' + ) diff --git a/events/views.py b/events/views.py index c509dbd04..b9f8db9d5 100644 --- a/events/views.py +++ b/events/views.py @@ -12,6 +12,10 @@ def get_batches(request): from django.core.exceptions import PermissionDenied +from django.core.exceptions import PermissionDenied +from django.views.decorators.csrf import csrf_protect +from django.db import connection + from django.contrib.auth.decorators import login_required from django.contrib.auth.models import Group from django.contrib import messages @@ -20,9 +24,8 @@ def get_batches(request): from django.template import RequestContext from django.http import HttpResponse, HttpResponseRedirect from django.shortcuts import get_object_or_404 -from django.http import HttpResponse, HttpResponseBadRequest +from django.http import HttpResponse from django.views.decorators.csrf import csrf_exempt - from django.http import Http404 from django.db.models import Q from django.db import IntegrityError @@ -71,6 +74,12 @@ def get_batches(request): from PyPDF2 import PdfFileWriter, PdfFileReader from django.template.context_processors import csrf +from cron.tasks import ( + async_process_test_attendance, + async_test_post_save +) + + from io import StringIO, BytesIO @@ -1858,41 +1867,159 @@ def training_participant_ceritificate(request, wid, participant_id): return response +# @login_required +# def test_request(request, role, rid = None): +# ''' Test request by organiser ''' +# user = request.user +# if not (user.is_authenticated() and ( is_organiser(user) or is_resource_person(user) or is_event_manager(user))): +# raise PermissionDenied() +# context = {} +# form = TestForm(user = user) +# if rid: +# t = Test.objects.get(pk = rid) +# user = t.organiser.user +# form = TestForm(user = user, instance = t) +# context['instance'] = t +# if request.method == 'POST': +# form = TestForm(request.POST, user = user) +# if form.is_valid(): +# dateTime = request.POST['tdate'].split(' ') +# t = Test() +# if rid: +# t = Test.objects.get(pk = rid) +# else: +# print("New Test.............") +# t.organiser_id = user.organiser.id +# t.academic = user.organiser.academic +# t.test_category_id = request.POST['test_category'] + +# """if int(request.POST['test_category']) == 1: +# t.training_id = request.POST['workshop']""" +# if int(request.POST['test_category']) == 2: +# t.training_id = request.POST['training'] +# if int(request.POST['test_category']) == 3: +# t.training_id = None +# test_trainings = request.POST['training'] +# test_training_dept = t.training.department_id +# if request.POST['id_foss']: +# test_foss = request.POST['id_foss'] +# else: +# test_foss = t.training.course.foss_id + +# t.invigilator_id = request.POST['invigilator'] +# t.foss_id = test_foss +# t.tdate = dateTime[0] +# t.ttime = dateTime[1] +# error = 0 +# errmsg = "" +# try: +# t.save() +# except IntegrityError as e: +# print(f"Test creation failed for {user.email}: {e}") +# error = 1 +# errmsg = "Test already created" +# prev_test = Test.objects.filter(organiser = t.organiser_id, academic = t.academic, foss = t.foss_id, tdate = t.tdate, ttime = t.ttime) +# if prev_test: +# messages.error(request, "You have already scheduled "+ t.foss.foss + " Test on "+t.tdate + " "+ t.ttime + ". Please select some other time.") + +# if not error and t.id and t.training_id: +# tras = TrainingAttend.objects.filter(training=t.training) +# fossmdlcourse = get_fossmdlcourse(t.foss_id, fossmdlmap_id=t.training.fossmdlmap_id) +# # try: +# # fossmdlcourse = FossMdlCourses.objects.get(foss_id = t.foss_id) +# # except FossMdlCourses.MultipleObjectsReturned: +# # fossmdlcourse = FossMdlCourses.objects.get(id = t.training.fossmdlmap_id) +# for tra in tras: +# user = tra.student.user +# mdluser = get_moodle_user(tra.training.training_planner.academic_id, user.first_name, user.last_name, tra.student.gender, tra.student.user.email)# if it create user rest password for django user too + +# if mdluser: +# print("mdluser present", mdluser.id) +# try: +# instance = TestAttendance.objects.get(test_id=t.id, mdluser_id=mdluser.id) +# except Exception as e: +# print(e) +# instance = TestAttendance() +# instance.student_id = tra.student.id +# instance.test_id = t.id +# instance.mdluser_id = mdluser.id +# instance.mdlcourse_id = fossmdlcourse.mdlcourse_id +# instance.mdlquiz_id = fossmdlcourse.mdlquiz_id +# instance.mdlattempt_id = 0 +# instance.status = 0 +# instance.save() + +# print("test_attendance created for ",tra.student.id) +# else: +# print("mdluser not found for", user.email) +# error = 1 +# if not error: +# t.department.clear() +# t.department.add(test_training_dept) +# #update logs +# message = t.academic.institution_name+" has made a test request for "+t.foss.foss+" on "+t.tdate +# if rid: +# message = t.academic.institution_name+" has updated test for "+t.foss.foss+" on dated "+t.tdate +# update_events_log(user_id = user.id, role = 0, category = 1, category_id = t.id, academic = t.academic_id, status = 0) +# update_events_notification(user_id = user.id, role = 0, category = 1, category_id = t.id, academic = t.academic_id, status = 0, message = message) + +# return HttpResponseRedirect("/software-training/test/"+role+"/pending/") +# messages.info(request, """ +# +# """) +# context['role'] = role +# context['status'] = 'request' +# context.update(csrf(request)) +# context['form'] = form +# return render(request, 'events/templates/test/form.html', context) + @login_required def test_request(request, role, rid = None): ''' Test request by organiser ''' + user = request.user if not (user.is_authenticated() and ( is_organiser(user) or is_resource_person(user) or is_event_manager(user))): raise PermissionDenied() context = {} form = TestForm(user = user) + if rid: t = Test.objects.get(pk = rid) user = t.organiser.user form = TestForm(user = user, instance = t) context['instance'] = t + if request.method == 'POST': form = TestForm(request.POST, user = user) + if form.is_valid(): dateTime = request.POST['tdate'].split(' ') - t = Test() + error = 0 + if rid: t = Test.objects.get(pk = rid) else: - print("New Test.............") + t = Test() t.organiser_id = user.organiser.id t.academic = user.organiser.academic + t.test_category_id = request.POST['test_category'] - """if int(request.POST['test_category']) == 1: - t.training_id = request.POST['workshop']""" if int(request.POST['test_category']) == 2: t.training_id = request.POST['training'] - if int(request.POST['test_category']) == 3: + else: t.training_id = None - test_trainings = request.POST['training'] - test_training_dept = t.training.department_id - if request.POST['id_foss']: + + test_training_dept = None + if t.training_id: + test_training_dept = t.training.department_id + + if request.POST.get('id_foss'): test_foss = request.POST['id_foss'] else: test_foss = t.training.course.foss_id @@ -1901,74 +2028,62 @@ def test_request(request, role, rid = None): t.foss_id = test_foss t.tdate = dateTime[0] t.ttime = dateTime[1] - error = 0 - errmsg = "" + try: t.save() - except IntegrityError as e: - print(f"Test creation failed for {user.email}: {e}") + except IntegrityError: error = 1 - errmsg = "Test already created" - prev_test = Test.objects.filter(organiser = t.organiser_id, academic = t.academic, foss = t.foss_id, tdate = t.tdate, ttime = t.ttime) - if prev_test: - messages.error(request, "You have already scheduled "+ t.foss.foss + " Test on "+t.tdate + " "+ t.ttime + ". Please select some other time.") - - if not error and t.id and t.training_id: - tras = TrainingAttend.objects.filter(training=t.training) - fossmdlcourse = get_fossmdlcourse(t.foss_id, fossmdlmap_id=t.training.fossmdlmap_id) - # try: - # fossmdlcourse = FossMdlCourses.objects.get(foss_id = t.foss_id) - # except FossMdlCourses.MultipleObjectsReturned: - # fossmdlcourse = FossMdlCourses.objects.get(id = t.training.fossmdlmap_id) - for tra in tras: - user = tra.student.user - mdluser = get_moodle_user(tra.training.training_planner.academic_id, user.first_name, user.last_name, tra.student.gender, tra.student.user.email)# if it create user rest password for django user too - - if mdluser: - print("mdluser present", mdluser.id) - try: - instance = TestAttendance.objects.get(test_id=t.id, mdluser_id=mdluser.id) - except Exception as e: - print(e) - instance = TestAttendance() - instance.student_id = tra.student.id - instance.test_id = t.id - instance.mdluser_id = mdluser.id - instance.mdlcourse_id = fossmdlcourse.mdlcourse_id - instance.mdlquiz_id = fossmdlcourse.mdlquiz_id - instance.mdlattempt_id = 0 - instance.status = 0 - instance.save() - - print("test_attendance created for ",tra.student.id) - else: - print("mdluser not found for", user.email) - error = 1 + messages.error(request, "You have already scheduled "+ t.foss.foss + " Test on "+t.tdate + " "+ t.ttime + ". Please select some other time.") + if not error: - t.department.clear() - t.department.add(test_training_dept) - #update logs - message = t.academic.institution_name+" has made a test request for "+t.foss.foss+" on "+t.tdate + # 🔥 async attendance + if t.training_id: + async_process_test_attendance(t) + + # 🔥 faster M2M update + if test_training_dept: + t.department.set([test_training_dept]) + else: + t.department.clear() + + message = ( + t.academic.institution_name + + " has made a test request for " + + t.foss.foss + " on " + t.tdate + ) if rid: - message = t.academic.institution_name+" has updated test for "+t.foss.foss+" on dated "+t.tdate - update_events_log(user_id = user.id, role = 0, category = 1, category_id = t.id, academic = t.academic_id, status = 0) - update_events_notification(user_id = user.id, role = 0, category = 1, category_id = t.id, academic = t.academic_id, status = 0, message = message) + message = ( + t.academic.institution_name + + " has updated test for " + + t.foss.foss + " dated " + t.tdate + ) + + # 🔥 async logs & notifications + async_test_post_save(t, user, message) + + return HttpResponseRedirect( + "/software-training/test/{}/pending/".format(role) + ) - return HttpResponseRedirect("/software-training/test/"+role+"/pending/") messages.info(request, """ """) + context['role'] = role context['status'] = 'request' context.update(csrf(request)) context['form'] = form return render(request, 'events/templates/test/form.html', context) + + + + @login_required def test_list(request, role, status): """ Organiser test index page """ From 0e32ba0d855ba7d05bdd3b33bcfaa6f0ba5372f2 Mon Sep 17 00:00:00 2001 From: ayishanishana21 Date: Thu, 22 Jan 2026 13:18:56 +0530 Subject: [PATCH 2/3] removed comments --- events/views.py | 110 ------------------------------------------------ 1 file changed, 110 deletions(-) diff --git a/events/views.py b/events/views.py index b9f8db9d5..03697988a 100644 --- a/events/views.py +++ b/events/views.py @@ -1867,116 +1867,6 @@ def training_participant_ceritificate(request, wid, participant_id): return response -# @login_required -# def test_request(request, role, rid = None): -# ''' Test request by organiser ''' -# user = request.user -# if not (user.is_authenticated() and ( is_organiser(user) or is_resource_person(user) or is_event_manager(user))): -# raise PermissionDenied() -# context = {} -# form = TestForm(user = user) -# if rid: -# t = Test.objects.get(pk = rid) -# user = t.organiser.user -# form = TestForm(user = user, instance = t) -# context['instance'] = t -# if request.method == 'POST': -# form = TestForm(request.POST, user = user) -# if form.is_valid(): -# dateTime = request.POST['tdate'].split(' ') -# t = Test() -# if rid: -# t = Test.objects.get(pk = rid) -# else: -# print("New Test.............") -# t.organiser_id = user.organiser.id -# t.academic = user.organiser.academic -# t.test_category_id = request.POST['test_category'] - -# """if int(request.POST['test_category']) == 1: -# t.training_id = request.POST['workshop']""" -# if int(request.POST['test_category']) == 2: -# t.training_id = request.POST['training'] -# if int(request.POST['test_category']) == 3: -# t.training_id = None -# test_trainings = request.POST['training'] -# test_training_dept = t.training.department_id -# if request.POST['id_foss']: -# test_foss = request.POST['id_foss'] -# else: -# test_foss = t.training.course.foss_id - -# t.invigilator_id = request.POST['invigilator'] -# t.foss_id = test_foss -# t.tdate = dateTime[0] -# t.ttime = dateTime[1] -# error = 0 -# errmsg = "" -# try: -# t.save() -# except IntegrityError as e: -# print(f"Test creation failed for {user.email}: {e}") -# error = 1 -# errmsg = "Test already created" -# prev_test = Test.objects.filter(organiser = t.organiser_id, academic = t.academic, foss = t.foss_id, tdate = t.tdate, ttime = t.ttime) -# if prev_test: -# messages.error(request, "You have already scheduled "+ t.foss.foss + " Test on "+t.tdate + " "+ t.ttime + ". Please select some other time.") - -# if not error and t.id and t.training_id: -# tras = TrainingAttend.objects.filter(training=t.training) -# fossmdlcourse = get_fossmdlcourse(t.foss_id, fossmdlmap_id=t.training.fossmdlmap_id) -# # try: -# # fossmdlcourse = FossMdlCourses.objects.get(foss_id = t.foss_id) -# # except FossMdlCourses.MultipleObjectsReturned: -# # fossmdlcourse = FossMdlCourses.objects.get(id = t.training.fossmdlmap_id) -# for tra in tras: -# user = tra.student.user -# mdluser = get_moodle_user(tra.training.training_planner.academic_id, user.first_name, user.last_name, tra.student.gender, tra.student.user.email)# if it create user rest password for django user too - -# if mdluser: -# print("mdluser present", mdluser.id) -# try: -# instance = TestAttendance.objects.get(test_id=t.id, mdluser_id=mdluser.id) -# except Exception as e: -# print(e) -# instance = TestAttendance() -# instance.student_id = tra.student.id -# instance.test_id = t.id -# instance.mdluser_id = mdluser.id -# instance.mdlcourse_id = fossmdlcourse.mdlcourse_id -# instance.mdlquiz_id = fossmdlcourse.mdlquiz_id -# instance.mdlattempt_id = 0 -# instance.status = 0 -# instance.save() - -# print("test_attendance created for ",tra.student.id) -# else: -# print("mdluser not found for", user.email) -# error = 1 -# if not error: -# t.department.clear() -# t.department.add(test_training_dept) -# #update logs -# message = t.academic.institution_name+" has made a test request for "+t.foss.foss+" on "+t.tdate -# if rid: -# message = t.academic.institution_name+" has updated test for "+t.foss.foss+" on dated "+t.tdate -# update_events_log(user_id = user.id, role = 0, category = 1, category_id = t.id, academic = t.academic_id, status = 0) -# update_events_notification(user_id = user.id, role = 0, category = 1, category_id = t.id, academic = t.academic_id, status = 0, message = message) - -# return HttpResponseRedirect("/software-training/test/"+role+"/pending/") -# messages.info(request, """ -# -# """) -# context['role'] = role -# context['status'] = 'request' -# context.update(csrf(request)) -# context['form'] = form -# return render(request, 'events/templates/test/form.html', context) @login_required def test_request(request, role, rid = None): From bb5fa11d30755012c65323ea72ce7eb582a3c27f Mon Sep 17 00:00:00 2001 From: ankitamk14 Date: Fri, 30 Jan 2026 19:52:22 +0530 Subject: [PATCH 3/3] added meta messages to the job --- cron/tasks.py | 100 ++++++++++++++++++++++++++++++++++++++++++++---- events/views.py | 6 +-- 2 files changed, 96 insertions(+), 10 deletions(-) diff --git a/cron/tasks.py b/cron/tasks.py index 68e189b9f..45d5c91f7 100644 --- a/cron/tasks.py +++ b/cron/tasks.py @@ -187,18 +187,40 @@ def process_test_attendance(test_id): """ close_old_connections() job = get_current_job() - if job: - job.meta['test_id'] = test_id + + print(f"\033[92m job ****** {job} \033[0m") + def meta_update(**updates): + """Update RQ job meta safely""" + if not job: + return + job.meta.update(updates) job.save_meta() + meta_update( + test_id=test_id, + status="starting", + started_at=int(time.time()), + progress_total=0, + progress_processed=0, + progress_pct=0, + stats_created_attendance=0, + stats_skipped_existing=0, + stats_missing_moodle=0, + message="Starting attendance processing.." + ) + try: test = Test.objects.select_related('training', 'foss').get(pk=test_id) except Test.DoesNotExist: + meta_update(status="done", message="Test not found. Exiting") return if not test.training_id: + meta_update(status="done", message="No training attached to test. Exiting.") return + meta_update(status="running", message="Loading training attendees...") + tras = TrainingAttend.objects.select_related( 'student__user', 'training__training_planner' @@ -209,15 +231,27 @@ def process_test_attendance(test_id): fossmdlmap_id=test.training.fossmdlmap_id ) - existing = { - (x.student_id, x.mdluser_id) - for x in TestAttendance.objects.filter(test=test) - } + # total count for progress (1 extra query; useful for dashboard) + total = tras.count() + meta_update(progress_total=total, progress_processed=0, progress_pct=0) + + existing = set( + TestAttendance.objects.filter(test=test) + .values_list("student_id", "mdluser_id") + ) mdluser_cache = {} new_rows = [] - for tra in tras: + processed = 0 + skipped_existing = 0 + missing_moodle = 0 + + # Update job meta every N rows to avoid excessive Redis writes + UPDATE_EVERY = 10 + meta_update(message=f"Processing {total} attendees...") + + for tra in tras.iterator(): user = tra.student.user key = ( @@ -233,10 +267,35 @@ def process_test_attendance(test_id): mdluser = mdluser_cache[key] if not mdluser: + missing_moodle += 1 + processed += 1 + # progress update + if (processed % UPDATE_EVERY == 0) or (processed == total): + pct = int((processed * 100) / total) if total else 100 + meta_update( + progress_processed=processed, + progress_pct=pct, + stats_created_attendance=len(new_rows), + stats_skipped_existing=skipped_existing, + stats_missing_moodle=missing_moodle, + message=f"Processing... ({processed}/{total})", + ) continue pair = (tra.student.id, mdluser.id) if pair in existing: + skipped_existing += 1 + processed += 1 + if (processed % UPDATE_EVERY == 0) or (processed == total): + pct = int((processed * 100) / total) if total else 100 + meta_update( + progress_processed=processed, + progress_pct=pct, + stats_created_attendance=len(new_rows), + stats_skipped_existing=skipped_existing, + stats_missing_moodle=missing_moodle, + message=f"Processing... ({processed}/{total})", + ) continue new_rows.append( @@ -250,12 +309,38 @@ def process_test_attendance(test_id): status=0 ) ) + processed += 1 + + if (processed % UPDATE_EVERY == 0) or (processed == total): + pct = int((processed * 100) / total) if total else 100 + meta_update( + progress_processed=processed, + progress_pct=pct, + stats_created_attendance=len(new_rows), + stats_skipped_existing=skipped_existing, + stats_missing_moodle=missing_moodle, + message=f"Processing... ({processed}/{total})", + ) + + # --- write phase --- + meta_update(status="writing", message=f"Writing {len(new_rows)} new TestAttendance rows...") if new_rows: close_old_connections() with transaction.atomic(): TestAttendance.objects.bulk_create(new_rows) + meta_update( + status="done", + finished_at=int(time.time()), + stats_created_attendance=len(new_rows), + stats_skipped_existing=skipped_existing, + stats_missing_moodle=missing_moodle, + progress_processed=total, + progress_pct=100, + message="Done.", + ) + def process_test_post_save(test_id, user_id, message,academic_id): """ @@ -304,6 +389,7 @@ def async_test_post_save(test, user, message): test.pk, user.pk, message, + test.academic_id, job_id="test_post_save_%s" % test.pk, job_timeout='24h' ) diff --git a/events/views.py b/events/views.py index 03697988a..19f31b708 100644 --- a/events/views.py +++ b/events/views.py @@ -1926,11 +1926,11 @@ def test_request(request, role, rid = None): messages.error(request, "You have already scheduled "+ t.foss.foss + " Test on "+t.tdate + " "+ t.ttime + ". Please select some other time.") if not error: - # 🔥 async attendance + # async attendance if t.training_id: async_process_test_attendance(t) - # 🔥 faster M2M update + # faster M2M update if test_training_dept: t.department.set([test_training_dept]) else: @@ -1948,7 +1948,7 @@ def test_request(request, role, rid = None): t.foss.foss + " dated " + t.tdate ) - # 🔥 async logs & notifications + # async logs & notifications async_test_post_save(t, user, message) return HttpResponseRedirect(