REFACTOR: Removes report data from schema.BaseSchema in favor of models.Report

* Removes the `schemas` module.
* Adds a new `report_schemas` package.
* Adds three abstract classes for report schemas:
	* `ReportSchema`: Base of all other schemas
	* `GenericReportSchema`: Base of Out-of-Band Reporting API incident reports.
	* `LegacyReportSchema`: Base for legacy incident reports.
* Adds a registry for report schemas.
* Adds modules to implement CSP and HPKP schemas in both contemporary and legacy arrangements.
* Adds a `fallback` module for Out-of-Band Reporting API incident reports that don't match any known schemas (preparation for #17).
* Updates other app components accordingly.

Closes #15

Author: rspeed

README.md

@@ -60,32 +60,6 @@ Run the database migrations:
 ./manage.py migrate lookout
 ```
 
-### Step 4
-
-Configure logging in `settings.py`.
-
-```python
-LOGGING = {
-	...
-	'handlers': {
-		...
-		'lookout_db': {
-			'class': 'lookout.logging.DatabaseLogHandler'
-		}
-	},
-	...
-	'loggers': {
-		...
-		'lookout': {
-			'handlers': ['lookout_db'],
-			'propagate': False
-		}
-	}
-}
-```
-
-The class `lookout.logging.DatabaseLogHandler` creates instances of `lookout.models.Report`. This will probably be getting replaced.
-
 
 ## Useful Guides
 

lookout/admin.py

@@ -30,21 +30,21 @@ def capitalization (word, index):
 
 @admin.register(Report)
 class ReportAdmin(admin.ModelAdmin):
-	date_hierarchy = 'created'
+	date_hierarchy = 'created_time'
 
 	empty_value_display = '<i>[empty]</i>'
 
-	list_display = ['created', 'type']
-	list_filter = ['created', 'type', 'generated']
+	list_display = ['created_time', 'type']
+	list_filter = ['created_time', 'incident_time', 'type']
 
 	save_on_top = True
 	actions = None
 
 	fieldsets = [
 		[None, {
-			'fields': ['created', 'generated', 'url']
+			'fields': ['created_time', 'incident_time', 'type', 'url']
 		}],
-		["Body", {
+		["Details", {
 			'description': "The report's full contents.",
 			'fields': ['pretty_body'],
 		}]
@@ -53,8 +53,11 @@ class ReportAdmin(admin.ModelAdmin):
 
 	def get_readonly_fields (self, request, obj=None):
 		""" Marks all fields as read-only. """
-		fields = self.fields or [f.name for f in self.model._meta.fields]
-		return fields + ['pretty_body']
+		# Don't use self.get_fields, as that causes an infinite recursion
+		fields = self.fields or []
+		for fieldset in self.fieldsets or []:
+			fields.extend(fieldset[1].get('fields', []))
+		return fields
 
 
 	def has_add_permission (self, request):

lookout/logging.py

@@ -1,7 +1,5 @@
-from logging import Handler
 
-
-__all__ = ['ReportMessage', 'DatabaseLogHandler']
+__all__ = ['ReportMessage']
 
 
 
@@ -20,28 +18,7 @@ def __init__ (self, msg=None, report=None):
 			self.report = report
 
 
-
 	def __str__ (self):
 		""" Returns a plaintext log entry that includes the most important data. """
 
 		return '{}\n{}'.format(self.msg, self.report)
-
-
-
-class DatabaseLogHandler (Handler):
-	""" Inserts standard reports into the database. """
-
-	def emit (self, record):
-		from .models import Report
-
-		try:
-			# Attempt to retrieve the report from the message
-			report = record.msg.report
-		except AttributeError:
-			# Don't do anything if it isn't a ReportMessage object
-			# TODO Raise an exception
-			return
-		else:
-			# Create the model instance
-			Report.objects.create_from_schema(report)
-

lookout/migrations/0002_auto_20171008_2238.py

@@ -0,0 +1,58 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+	dependencies = [
+		('lookout', '0001_initial'),
+	]
+
+	operations = [
+		migrations.RenameField(
+			model_name='report',
+			old_name='created',
+			new_name='created_time'
+		),
+		migrations.AlterField(
+			model_name='report',
+			name='created_time',
+			field=models.DateTimeField(
+				primary_key=True, serialize=False, auto_now_add=True,
+				help_text='When the incident report was submitted.',
+				verbose_name='Submission Time'
+			)
+		),
+		migrations.RenameField(
+			model_name='report',
+			old_name='generated',
+			new_name='incident_time'
+		),
+		migrations.AlterField(
+			model_name='report',
+			name='incident_time',
+			field=models.DateTimeField(db_index=True, help_text='When the incident occurred.')
+		),
+		migrations.AlterField(
+			model_name='report',
+			name='type',
+			field=models.CharField(
+				choices=[
+					('misc', 'Generic HTTP Reporting API incident report'),
+					('csp', 'Content Security Policy Report'),
+					('hpkp', 'HTTP Public Key Pinning Report')
+				],
+				db_index=True,
+				help_text="The report's category.",
+				max_length=120
+			)
+		),
+		migrations.AlterField(
+			model_name='report',
+			name='body',
+			field=models.TextField(help_text='The contents of the incident report.'),
+		),
+		migrations.AlterModelOptions(
+			name='report',
+			options={'ordering': ['-incident_time']},
+		),
+	]

lookout/migrations/0003_auto_20171008_2324.py

@@ -0,0 +1,22 @@
+from django.db import migrations, models
+import uuid
+
+
+class Migration(migrations.Migration):
+
+	dependencies = [
+		('lookout', '0002_auto_20171008_2238'),
+	]
+
+	operations = [
+		migrations.AddField(
+			model_name='report',
+			name='uuid',
+			field=models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False, editable=False),
+		),
+		migrations.AlterField(
+			model_name='report',
+			name='created_time',
+			field=models.DateTimeField(auto_now_add=True, db_index=True, help_text='When the incident report was submitted.', verbose_name='Submission Time'),
+		),
+	]

lookout/models.py

@@ -1,5 +1,6 @@
 import logging
 import json
+import uuid
 
 from datetime import timedelta
 
@@ -12,6 +13,8 @@
 from pygments.lexers.data import JsonLexer
 from pygments.formatters.html import HtmlFormatter
 
+from .report_schemas import get_matching_schema, report_schema_registry
+
 
 logger = logging.getLogger(__name__)
 
@@ -21,39 +24,58 @@
 
 
 class ReportManager (models.Manager):
-	def create_from_schema (self, report):
-		""" Converts a parsed JSON report into a model instance. """
-
-		# Report data as a dictionary
-		data = dict(report)
-
-		# Use a static datetime object to make sure `created`, `generated`, and `body['age']` are consistent
-		now = timezone.now()
-
-		# Build the model instance
-		return self.create(
-			created=now,
-			type=data['type'],
-			# Use the report's `age` property to determine when it was generated
-			generated=now - timedelta(milliseconds=data['age']),
-			url=data['url'],
-			# Store the serialized version
-			body=str(report)
-		)
+	def create_from_json (self, report_json):
+		""" Converts JSON data into a list of Report instances. """
+
+		logger.debug("Decoding JSON")
+		report_datum = json.loads(report_json)
+
+		# Wrap single reports in a list
+		if not isinstance(report_datum, list):
+			report_datum = [report_datum]
+
+		# Iterate over separate reports
+		for report_data in report_datum:
+			logger.debug("Attempt to determine the type of report by testing each schema.")
+
+			# Figure out what type of report it is
+			schema = get_matching_schema(report_data)
+
+			# Normalize to a generic schema
+			schema, report_data = schema.normalize(report_data)
+
+			# Use a static datetime object to make sure `created_time`, `incident_time`, and `body['age']` are consistent
+			now = timezone.now()
+
+			# Build the model instance
+			yield self.create(
+				created_time=now,
+				# Use the report's `age` property to determine when the incident occurred
+				incident_time=now - timedelta(milliseconds=report_data['age']),
+				type=schema.type,
+				url=report_data['url'],
+				body=json.dumps(report_data)
+			)
+
 
+report_types = [(schema.type, schema.name) for schema in report_schema_registry.values()]
 
 
 class Report (models.Model):
 	""" A report filed through the HTTP Reporting API. """
 
 	objects = ReportManager()
 
-	created = models.DateTimeField(auto_now_add=True, primary_key=True)
-
-	type = models.CharField(max_length=120, help_text="The report's category.")
-	generated = models.DateTimeField(help_text="The time at which the report was generated.")
+	uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+	created_time = models.DateTimeField(auto_now_add=True, db_index=True, verbose_name="Submission Time", help_text="When the incident report was submitted.")
+	incident_time = models.DateTimeField(db_index=True, help_text="When the incident occurred.")
+	type = models.CharField(max_length=120, db_index=True, choices=report_types, help_text="The report's category.")
 	url = models.URLField(help_text="The address of the document or worker from which the report was generated.")
-	body = models.TextField(help_text="The contents of the report.")
+	body = models.TextField(help_text="The contents of the incident report.")
+
+
+	class Meta:
+		ordering = ['-incident_time']
 
 
 	def pretty_body (self):
@@ -68,12 +90,14 @@ def pretty_body (self):
 		return mark_safe(response)
 
 
-
-	class Meta:
-		ordering = ['-created']
+	@property
+	def schema (self):
+		return report_schema_registry.get(self.type)
 
 
 	def __str__ (self):
-		return "{} report from {}".format(self.type.capitalize(), formats.date_format(self.created, 'SHORT_DATETIME_FORMAT'))
-
-
+		""" Something like " """
+		return "{} report from {}".format(
+			self.schema.name,
+			formats.date_format(self.incident_time, 'SHORT_DATETIME_FORMAT')
+		)

lookout/report_schemas/__init__.py

@@ -0,0 +1,4 @@
+from .base import *
+from .csp import *
+from .hpkp import *
+from .fallback import *