Pushing the ORMto its limits

👋 Hello, I’m SigurdDeveloper at Kolonial.no • github.com/ljodal/djangcon-eu-2019

Agenda

• Some tips and tricks

• Subqueries

• Constraints and indexes

• Window functions

• Customizing the ORM

Disclaimer:This talk is quite code heavy!

Useful tips and tricks

CustomQuerySet

and Manager

class OrderManager(models.Manager): def create_order( self, customer, products ): ...

class OrderQuerySet(QuerySet): def undelivered(self): return self.filter(is_delivered=False)

class Order(models.Model): ... objects = OrderManager.from_queryset( OrderQuerySet )()

Order.objects.create_order(customer=...) Order.objects.undelivered()

Inspecting queries

>>> orders = Order.objects.all() <OrderQuerySet ...>

>>> str(orders.query) SELECT ... FROM «orders_order"

>>> print(order.explain(verbose=True)) Seq Scan on public.orders_order (cost=0.00..28.10 rows=1810 width=17) Output: id, customer_id, created_at, is_shipped

Avoiding extra

queries

for order in Order.objects.all(): # This triggers one query per order print(order.customer.name) # This also triggers one query per order for line in order.lines.all(): print(line)

Order.objects.select_related('customer')

Order.objects.prefetch_related('lines')

Avoiding race conditions

with transaction.atomic(): product = ( Product.objects .select_for_update() .get(id=1) ) product.inventory -= 1 product.save()

Subqueries

Latest order

customers = Customer.objects.annotate( latest_order_time=Subquery( Order.objects.filter( customer=OuterRef('pk'), ).order_by( '-created_at' ).values( 'created_at' )[:1] ) )

>>> customers.first().latest_order_time 1

With aggregation

budgets = SalesTarget.objects.annotate( gross_total_sales=Subquery( Order.objects.filter( created_at__year=OuterRef('year'), created_at__month=OuterRef('month'), ).values_list( ExtractYear('created_at'), ExtractMonth('created_at') ).annotate( gross_total=Sum('lines__gross_amount'), ).value_lists( 'gross_total', ) ), )

>>> budgets.first().gross_total_sales 12.00

Aggregation without

grouping

id | week_day | lines__gross_amount 1 | 7 | 7.5 2 | 1 | 2.5 3 | 3 | 2.0

targets = SalesTarget.objects.annotate( weekend_revenue=Subquery( Order.objects.filter( created_at__week_day__in=[7, 1], ).values_list( Sum(‘lines__gross_amount'), ) ), )

>>> targest.first().weekend_revenue 7.50 # Oops, this is not what we wanted

Aggregation without

grouping

id | week_day | lines__gross_amount 1 | 7 | 7.5 2 | 1 | 2.5 3 | 3 | 2.0

targets = SalesTargets.objects.annotate( weekend_revenue=Subquery( Order.objects.filter( created_at__week_day__in=[7, 1], ).values_list( Func( 'lines__gross_amount', function='SUM', ), ) ), )

>>> targets.first().weekend_revenue 10.00

Custom constraints and indexes

Unique constraints

class SalesTarget(Model): year = models.IntegerField() month = models.IntegerField() target = models.DecimalField(...)

class Meta:

unique_together = [ ('year', 'month'), ]

Partial unique

constraint

class Order(Model): ...

class Meta:

constraints = [ UniqueConstraint( name='limit_pending_orders', fields=['customer', 'is_shipped'], condition=Q(is_shipped=False), ) ]

Check constraint

class MonthlyBudget(Model): ...

class Meta:

constraints = [ CheckConstraint( check=Q(month__in=range(1, 13)), name='check_valid_month', ) ]

Partial index

class Order(Model): ...

class Meta:

indexes = [ Index( name='unshipped_orders', fields=['pk', ], condition=Q(is_shipped=False), ) ]

Window functions

Previous order from

same customer

orders = Order.objects.annotate( prev_order_id=Window( expression=Lag('order_id', 1), partition_by=[F('customer_id')], order_by=F('created_at').asc(), ), )

>>> orders.first().prev_order_id 1

Extending with custom functionality

Custom functions

class Round(Func): function = 'ROUND'

A more complex custom function

class AsDateTime(Func):

arity = 2 output_field = DateTimeField()

def as_postgresql( self, compiler, connection, **extra_context ):

extra_context['tz'] = settings.TIME_ZONE template = ( "(%(expressions)s || '%(tz)s')::timestamptz" )

return self.as_sql( compiler, connection, arg_joiner='+', template=template, **extra_context )

qs.annotate( datetime=AsDateTime('date_field', 'time_field') )

Writing custom SQL

Order.objects.annotate( age=RawSQL('age(created_at)'), )

Order.objects.extra( select={ 'age': 'age(created_at)', }, )

Order.objects.raw( ''' SELECT *, age(created_at) as age FROM orders_order ''' )

with connection.cursor() as cursor: cursor.execute('SELECT 2') cursor.fetchone()

There’s so much moreTake a look at the Django documentation

Thanks!@sigurdlj • github.com/ljodal/djangocon-eu-2019

medium.com/kolonial-no-product-tech