import collections
import datetime
import decimal
import json
import logging
import os
from dateutil.relativedelta import relativedelta
from django.conf import settings
from django.contrib.contenttypes.fields import GenericRelation
from django.contrib.humanize.templatetags.humanize import ordinal
from django.contrib.postgres.fields import JSONField
from django.core.exceptions import ValidationError
from django.core.validators import MinValueValidator
from django.db import models
from django.db.models import (
Case,
F,
ExpressionWrapper,
Max,
OuterRef,
Q,
Subquery,
Sum,
Value as V,
When,
)
from django.db.models.functions import Cast, Coalesce
from django.db.models.signals import post_delete
from django.dispatch.dispatcher import receiver
from django.urls import reverse
from django.utils import timezone
from django.utils.functional import cached_property
from django.utils.translation import ugettext as _
from wagtail.contrib.settings.models import BaseSetting, register_setting
from wagtail.admin.edit_handlers import (
FieldPanel,
StreamFieldPanel,
)
from wagtail.core.fields import StreamField
from opentech.apply.funds.models.mixins import AccessFormData
from opentech.apply.stream_forms.blocks import FormFieldsBlock
from opentech.apply.stream_forms.files import StreamFieldDataEncoder
from opentech.apply.stream_forms.models import BaseStreamForm
from addressfield.fields import ADDRESS_FIELDS_ORDER
from opentech.apply.activity.messaging import MESSAGES, messenger
from opentech.apply.utils.storage import PrivateStorage
logger = logging.getLogger(__name__)
def contract_path(instance, filename):
return f'projects/{instance.project_id}/contracts/{filename}'
def document_path(instance, filename):
return f'projects/{instance.project_id}/supporting_documents/{filename}'
def invoice_path(instance, filename):
return f'projects/{instance.project_id}/payment_invoices/{filename}'
def receipt_path(instance, filename):
return f'projects/{instance.payment_request.project_id}/payment_receipts/{filename}'
def report_path(instance, filename):
return f'reports/{instance.report.report_id}/version/{instance.report_id}/{filename}'
class Approval(models.Model):
project = models.ForeignKey("Project", on_delete=models.CASCADE, related_name="approvals")
by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="approvals")
created_at = models.DateTimeField(auto_now_add=True)
class Meta:
unique_together = ['project', 'by']
def __str__(self):
return f'Approval of "{self.project.title}" by {self.by}'
class ContractQuerySet(models.QuerySet):
def approved(self):
return self.filter(is_signed=True, approver__isnull=False)
class Contract(models.Model):
approver = models.ForeignKey(settings.AUTH_USER_MODEL, null=True, on_delete=models.SET_NULL, related_name='contracts')
project = models.ForeignKey("Project", on_delete=models.CASCADE, related_name="contracts")
file = models.FileField(upload_to=contract_path, storage=PrivateStorage())
is_signed = models.BooleanField("Signed?", default=False)
created_at = models.DateTimeField(auto_now_add=True)
approved_at = models.DateTimeField(null=True)
objects = ContractQuerySet.as_manager()
@property
def state(self):
return 'Signed' if self.is_signed else 'Unsigned'
def __str__(self):
return f'Contract for {self.project} ({self.state})'
def get_absolute_url(self):
return reverse('apply:projects:contract', args=[self.project.pk, self.pk])
class PacketFile(models.Model):
category = models.ForeignKey("DocumentCategory", null=True, on_delete=models.CASCADE, related_name="packet_files")
project = models.ForeignKey("Project", on_delete=models.CASCADE, related_name="packet_files")
title = models.TextField()
document = models.FileField(upload_to=document_path, storage=PrivateStorage())
def __str__(self):
return f'Project file: {self.title}'
def get_remove_form(self):
"""
Get an instantiated RemoveDocumentForm with this class as `instance`.
This allows us to build instances of the RemoveDocumentForm for each
instance of PacketFile in the supporting documents template. The
standard Delegated View flow makes it difficult to create these forms
in the view or template.
"""
from .forms import RemoveDocumentForm
return RemoveDocumentForm(instance=self)
@receiver(post_delete, sender=PacketFile)
def delete_packetfile_file(sender, instance, **kwargs):
# Remove the file and don't save the base model
instance.document.delete(False)
class PaymentApproval(models.Model):
request = models.ForeignKey('PaymentRequest', on_delete=models.CASCADE, related_name="approvals")
by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="payment_approvals")
created_at = models.DateTimeField(auto_now_add=True)
def __str__(self):
return f'Approval for {self.request} by {self.by}'
class PaymentReceipt(models.Model):
payment_request = models.ForeignKey("PaymentRequest", on_delete=models.CASCADE, related_name="receipts")
file = models.FileField(upload_to=receipt_path, storage=PrivateStorage())
def __str__(self):
return os.path.basename(self.file.name)
SUBMITTED = 'submitted'
CHANGES_REQUESTED = 'changes_requested'
UNDER_REVIEW = 'under_review'
PAID = 'paid'
DECLINED = 'declined'
REQUEST_STATUS_CHOICES = [
(SUBMITTED, 'Submitted'),
(CHANGES_REQUESTED, 'Changes Requested'),
(UNDER_REVIEW, 'Under Review'),
(PAID, 'Paid'),
(DECLINED, 'Declined'),
]
class PaymentRequestQueryset(models.QuerySet):
def in_progress(self):
return self.exclude(status__in=[DECLINED, PAID])
def rejected(self):
return self.filter(status=DECLINED)
def not_rejected(self):
return self.exclude(status=DECLINED)
def total_value(self, field):
return self.aggregate(total=Coalesce(Sum(field), V(0)))['total']
def paid_value(self):
return self.filter(status=PAID).total_value('paid_value')
def unpaid_value(self):
return self.filter(status__in=[SUBMITTED, UNDER_REVIEW]).total_value('requested_value')
class PaymentRequest(models.Model):
project = models.ForeignKey("Project", on_delete=models.CASCADE, related_name="payment_requests")
by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="payment_requests")
requested_value = models.DecimalField(
default=0,
max_digits=10,
decimal_places=2,
validators=[MinValueValidator(decimal.Decimal('0.01'))],
)
paid_value = models.DecimalField(
max_digits=10,
decimal_places=2,
validators=[MinValueValidator(decimal.Decimal('0.01'))],
null=True
)
invoice = models.FileField(upload_to=invoice_path, storage=PrivateStorage())
requested_at = models.DateTimeField(auto_now_add=True)
date_from = models.DateTimeField()
date_to = models.DateTimeField()
comment = models.TextField(blank=True)
status = models.TextField(choices=REQUEST_STATUS_CHOICES, default=SUBMITTED)
objects = PaymentRequestQueryset.as_manager()
def __str__(self):
return f'Payment requested for {self.project}'
@property
def has_changes_requested(self):
return self.status == CHANGES_REQUESTED
@property
def status_display(self):
return self.get_status_display()
def can_user_delete(self, user):
if user.is_applicant:
if self.status in (SUBMITTED, CHANGES_REQUESTED):
return True
if user.is_apply_staff:
if self.status in {SUBMITTED}:
return True
return False
def can_user_edit(self, user):
if user.is_applicant:
if self.status in {SUBMITTED, CHANGES_REQUESTED}:
return True
if user.is_apply_staff:
if self.status in {SUBMITTED}:
return True
return False
def can_user_change_status(self, user):
if not user.is_apply_staff:
return False # Users can't change status
if self.status in {PAID, DECLINED}:
return False
return True
@property
def value(self):
return self.paid_value or self.requested_value
def get_absolute_url(self):
return reverse('apply:projects:payments:detail', args=[self.pk])
COMMITTED = 'committed'
CONTRACTING = 'contracting'
IN_PROGRESS = 'in_progress'
CLOSING = 'closing'
COMPLETE = 'complete'
PROJECT_STATUS_CHOICES = [
(COMMITTED, 'Committed'),
(CONTRACTING, 'Contracting'),
(IN_PROGRESS, 'In Progress'),
(CLOSING, 'Closing'),
(COMPLETE, 'Complete'),
]
class ProjectQuerySet(models.QuerySet):
def active(self):
"Projects that are not finished"
return self.exclude(status=COMPLETE)
def in_progress(self):
"Projects that users need to interact with, submitting reports or payment request"
return self.filter(
status__in=(IN_PROGRESS, CLOSING,)
)
def complete(self):
return self.filter(status=COMPLETE)
def in_approval(self):
return self.filter(
is_locked=True,
status=COMMITTED,
approvals__isnull=True,
)
def by_end_date(self, desc=False):
order = getattr(F('proposed_end'), 'desc' if desc else 'asc')(nulls_last=True)
return self.order_by(order)
def with_amount_paid(self):
return self.annotate(
amount_paid=Coalesce(Sum('payment_requests__paid_value'), V(0)),
)
def with_last_payment(self):
return self.annotate(
last_payment_request=Max('payment_requests__requested_at'),
)
def with_start_date(self):
return self.annotate(
start=Cast(
Subquery(
Contract.objects.filter(
project=OuterRef('pk'),
).approved().order_by(
'approved_at'
).values('approved_at')[:1]
),
models.DateField(),
)
)
def for_table(self):
return self.with_amount_paid().with_last_payment().select_related(
'report_config',
'submission__page',
'lead',
)
class Project(BaseStreamForm, AccessFormData, models.Model):
lead = models.ForeignKey(settings.AUTH_USER_MODEL, null=True, on_delete=models.SET_NULL, related_name='lead_projects')
submission = models.OneToOneField("funds.ApplicationSubmission", on_delete=models.CASCADE)
user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.SET_NULL, null=True, related_name='owned_projects')
title = models.TextField()
contact_legal_name = models.TextField(_('Person or Organisation name'), default='')
contact_email = models.TextField(_('Email'), default='')
contact_address = models.TextField(_('Address'), default='')
contact_phone = models.TextField(_('Phone'), default='')
value = models.DecimalField(
default=0,
max_digits=10,
decimal_places=2,
validators=[MinValueValidator(decimal.Decimal('0.01'))],
)
proposed_start = models.DateTimeField(_('Proposed Start Date'), null=True)
proposed_end = models.DateTimeField(_('Proposed End Date'), null=True)
status = models.TextField(choices=PROJECT_STATUS_CHOICES, default=COMMITTED)
form_data = JSONField(encoder=StreamFieldDataEncoder, default=dict)
form_fields = StreamField(FormFieldsBlock(), null=True)
# tracks read/write state of the Project
is_locked = models.BooleanField(default=False)
# tracks updates to the Projects fields via the Project Application Form.
user_has_updated_details = models.BooleanField(default=False)
activities = GenericRelation(
'activity.Activity',
content_type_field='source_content_type',
object_id_field='source_object_id',
related_query_name='project',
)
created_at = models.DateTimeField(auto_now_add=True)
sent_to_compliance_at = models.DateTimeField(null=True)
objects = ProjectQuerySet.as_manager()
def __str__(self):
return self.title
@property
def status_display(self):
return self.get_status_display()
def get_address_display(self):
try:
address = json.loads(self.contact_address)
except json.JSONDecodeError:
return ''
else:
return ', '.join(
address.get(field)
for field in ADDRESS_FIELDS_ORDER
if address.get(field)
)
@classmethod
def create_from_submission(cls, submission):
"""
Create a Project from the given submission.
Returns a new Project or the given ApplicationSubmissions existing
Project.
"""
if not settings.PROJECTS_ENABLED:
logging.error(f'Tried to create a Project for Submission ID={submission.id} while projects are disabled')
return None
# OneToOne relations on the targetted model cannot be accessed without
# an exception when the relation doesn't exist (is None). Since we
# want to fail fast here, we can use hasattr instead.
if hasattr(submission, 'project'):
return submission.project
return Project.objects.create(
submission=submission,
title=submission.title,
user=submission.user,
contact_email=submission.user.email,
contact_legal_name=submission.user.full_name,
contact_address=submission.form_data.get('address', ''),
value=submission.form_data.get('value', 0),
)
@property
def start_date(self):
# Assume project starts when OTF are happy with the first signed contract
first_approved_contract = self.contracts.approved().order_by('approved_at').first()
if not first_approved_contract:
return None
return first_approved_contract.approved_at.date()
@property
def end_date(self):
# Aiming for the proposed end date as the last day of the project
# If still ongoing assume today is the end
return max(
self.proposed_end.date(),
timezone.now().date(),
)
def paid_value(self):
return self.payment_requests.paid_value()
def unpaid_value(self):
return self.payment_requests.unpaid_value()
def clean(self):
if self.proposed_start is None:
return
if self.proposed_end is None:
return
if self.proposed_start > self.proposed_end:
raise ValidationError(_('Proposed End Date must be after Proposed Start Date'))
def save(self, *args, **kwargs):
creating = not self.pk
if creating:
files = self.extract_files()
else:
self.process_file_data(self.form_data)
super().save(*args, **kwargs)
if creating:
self.process_file_data(files)
def editable_by(self, user):
if self.editable:
return True
# Approver can edit it when they are approving
return user.is_approver and self.can_make_approval
@property
def editable(self):
if self.status not in (CONTRACTING, COMMITTED):
return True
# Someone has approved the project - consider it locked while with contracting
if self.approvals.exists():
return False
# Someone must lead the project to make changes
return self.lead and not self.is_locked
def get_absolute_url(self):
if settings.PROJECTS_ENABLED:
return reverse('apply:projects:detail', args=[self.id])
return '#'
@property
def can_make_approval(self):
return self.is_locked and self.status == COMMITTED
def can_request_funding(self):
"""
Should we show this Project's funding block?
"""
return self.status in (CLOSING, IN_PROGRESS)
@property
def can_send_for_approval(self):
"""
Wrapper to expose the pending approval state
We don't want to expose a "Sent for Approval" state to the end User so
we infer it from the current status being "Comitted" and the Project
being locked.
"""
correct_state = self.status == COMMITTED and not self.is_locked
return correct_state and self.user_has_updated_details
@property
def requires_approval(self):
return not self.approvals.exists()
def get_missing_document_categories(self):
"""
Get the number of documents required to meet each DocumentCategorys minimum
"""
# Count the number of documents in each category currently
existing_categories = DocumentCategory.objects.filter(packet_files__project=self)
counter = collections.Counter(existing_categories)
# Find the difference between the current count and recommended count
for category in DocumentCategory.objects.all():
current_count = counter[category]
difference = category.recommended_minimum - current_count
if difference > 0:
yield {
'category': category,
'difference': difference,
}
@property
def is_in_progress(self):
return self.status == IN_PROGRESS
def send_to_compliance(self, request):
"""Notify Compliance about this Project."""
messenger(
MESSAGES.SENT_TO_COMPLIANCE,
request=request,
user=request.user,
source=self,
)
self.sent_to_compliance_at = timezone.now()
self.save(update_fields=['sent_to_compliance_at'])
@register_setting
class ProjectSettings(BaseSetting):
compliance_email = models.TextField("Compliance Email")
class DocumentCategory(models.Model):
name = models.CharField(max_length=254)
recommended_minimum = models.PositiveIntegerField()
def __str__(self):
return self.name
class Meta:
ordering = ('name',)
verbose_name_plural = 'Document Categories'
class ProjectApprovalForm(BaseStreamForm, models.Model):
name = models.CharField(max_length=255)
form_fields = StreamField(FormFieldsBlock())
panels = [
FieldPanel('name'),
StreamFieldPanel('form_fields'),
]
def __str__(self):
return self.name
class ReportConfig(models.Model):
"""Persists configuration about the reporting schedule etc"""
WEEK = "week"
MONTH = "month"
FREQUENCY_CHOICES = [
(WEEK, "Weeks"),
(MONTH, "Months"),
]
project = models.OneToOneField("Project", on_delete=models.CASCADE, related_name="report_config")
schedule_start = models.DateField(null=True)
occurrence = models.PositiveSmallIntegerField(default=1)
frequency = models.CharField(choices=FREQUENCY_CHOICES, default=MONTH, max_length=5)
def get_frequency_display(self):
next_report = self.current_due_report()
if self.frequency == self.MONTH:
if self.schedule_start and self.schedule_start.day == 31:
day_of_month = 'last day'
else:
day_of_month = ordinal(next_report.end_date.day)
if self.occurrence == 1:
return f"Monthly on the { day_of_month } of the month"
return f"Every { self.occurrence } months on the { day_of_month } of the month"
weekday = next_report.end_date.strftime('%A')
if self.occurrence == 1:
return f"Every week on { weekday }"
return f"Every {self.occurrence} weeks on { weekday }"
def is_up_to_date(self):
return len(self.project.reports.to_do()) == 0
def outstanding_reports(self):
return len(self.project.reports.to_do())
def has_very_late_reports(self):
return self.project.reports.any_very_late()
def past_due_reports(self):
return self.project.reports.to_do()
def last_report(self):
today = timezone.now().date()
return self.project.reports.filter(
Q(end_date__lt=today) | Q(current__isnull=False)
).first()
def current_due_report(self):
# Project not started - no reporting required
if not self.project.start_date:
return None
today = timezone.now().date()
last_report = self.last_report()
schedule_date = self.schedule_start or self.project.start_date
if last_report:
if last_report.end_date < schedule_date:
# reporting schedule changed schedule_start is now the next report date
next_due_date = schedule_date
else:
# we've had a report since the schedule date so base next deadline from the report
next_due_date = self.next_date(last_report.end_date)
else:
# first report required
if self.schedule_start and self.schedule_start >= today:
# Schedule changed since project inception
next_due_date = self.schedule_start
else:
# schedule_start is the first day the project so the "last" period
# ended one day before that. If date is in past we required report now
next_due_date = max(
self.next_date(schedule_date - relativedelta(days=1)),
today,
)
report, _ = self.project.reports.update_or_create(
project=self.project,
end_date__gte=today,
current__isnull=True,
defaults={'end_date': next_due_date}
)
return report
def next_date(self, last_date):
delta_frequency = self.frequency + 's'
delta = relativedelta(**{delta_frequency: self.occurrence})
next_date = last_date + delta
return next_date
class ReportQueryset(models.QuerySet):
def done(self):
return self.filter(
Q(current__isnull=False) | Q(skipped=True),
)
def to_do(self):
today = timezone.now().date()
return self.filter(
current__isnull=True,
skipped=False,
end_date__lt=today,
).order_by('end_date')
def any_very_late(self):
two_weeks_ago = timezone.now().date() - relativedelta(weeks=2)
return self.to_do().filter(end_date__lte=two_weeks_ago)
def submitted(self):
return self.filter(current__isnull=False)
def for_table(self):
return self.annotate(
last_end_date=Subquery(
Report.objects.filter(
project=OuterRef('project_id'),
end_date__lt=OuterRef('end_date')
).values('end_date')[:1]
),
project_start_date=Subquery(
Project.objects.filter(
pk=OuterRef('project_id'),
).with_start_date().values('start')[:1]
),
start=Case(
When(
last_end_date__isnull=False,
# Expression Wrapper doesn't cast the calculated object
# Use cast to get an actual date object
then=Cast(
ExpressionWrapper(
F('last_end_date') + datetime.timedelta(days=1),
output_field=models.DateTimeField(),
),
models.DateField(),
),
),
default=F('project_start_date'),
output_field=models.DateField(),
)
)
class Report(models.Model):
skipped = models.BooleanField(default=False)
end_date = models.DateField()
project = models.ForeignKey("Project", on_delete=models.CASCADE, related_name="reports")
submitted = models.DateTimeField(null=True)
notified = models.DateTimeField(null=True)
current = models.OneToOneField(
"ReportVersion",
on_delete=models.CASCADE,
related_name='live_for_report',
null=True,
)
draft = models.OneToOneField(
"ReportVersion",
on_delete=models.CASCADE,
related_name='draft_for_report',
null=True,
)
objects = ReportQueryset.as_manager()
class Meta:
ordering = ('-end_date',)
def get_absolute_url(self):
return reverse('apply:projects:reports:detail', kwargs={'pk': self.pk})
@property
def past_due(self):
return timezone.now().date() > self.end_date
@property
def is_very_late(self):
two_weeks_ago = timezone.now().date() - relativedelta(weeks=2)
two_weeks_late = self.end_date < two_weeks_ago
not_submitted = not self.current
return not_submitted and two_weeks_late
@property
def can_submit(self):
return self.start_date <= timezone.now().date()
@property
def submitted_date(self):
if self.submitted:
return self.submitted.date()
@cached_property
def start_date(self):
last_report = self.project.reports.filter(end_date__lt=self.end_date).first()
if last_report:
return last_report.end_date + relativedelta(days=1)
return self.project.start_date
class ReportVersion(models.Model):
report = models.ForeignKey("Report", on_delete=models.CASCADE, related_name="versions")
submitted = models.DateTimeField()
public_content = models.TextField()
private_content = models.TextField()
draft = models.BooleanField()
author = models.ForeignKey(
settings.AUTH_USER_MODEL,
on_delete=models.SET_NULL,
related_name="reports",
null=True,
)
class ReportPrivateFiles(models.Model):
report = models.ForeignKey("ReportVersion", on_delete=models.CASCADE, related_name="files")
document = models.FileField(upload_to=report_path, storage=PrivateStorage())
@property
def filename(self):
return os.path.basename(self.document.name)
def __str__(self):
return self.filename
def get_absolute_url(self):
return reverse('apply:projects:reports:document', kwargs={'pk': self.report.report_id, 'file_pk': self.pk})