python - OperationalError, no such column. Django

Question

I am very new to django and was able to finish the tutorial on djangoproject.com without any errors. I am now going through the Django REST framework tutorial found at http://www.django-rest-framework.org/ I am almost finished with it and just added authentication. Now I am getting :

OperationalError at /snippets/
no such column: snippets_snippet.owner_id
Request Method: GET
Request URL:    http://localhost:8000/snippets/
Django Version: 1.7
Exception Type: OperationalError
Exception Value:    
no such column: snippets_snippet.owner_id
Exception Location: /Users/taylorallred/Desktop/env/lib/python2.7/site-packages/django/db/backends/sqlite3/base.py in execute, line 485
Python Executable:  /Users/taylorallred/Desktop/env/bin/python
Python Version: 2.7.5
Python Path:    
['/Users/taylorallred/Desktop/tutorial',
 '/Users/taylorallred/Desktop/env/lib/python27.zip',
 '/Users/taylorallred/Desktop/env/lib/python2.7',
 '/Users/taylorallred/Desktop/env/lib/python2.7/plat-darwin',
 '/Users/taylorallred/Desktop/env/lib/python2.7/plat-mac',
 '/Users/taylorallred/Desktop/env/lib/python2.7/plat-mac/lib-scriptpackages',
 '/Users/taylorallred/Desktop/env/Extras/lib/python',
 '/Users/taylorallred/Desktop/env/lib/python2.7/lib-tk',
 '/Users/taylorallred/Desktop/env/lib/python2.7/lib-old',
 '/Users/taylorallred/Desktop/env/lib/python2.7/lib-dynload',
 '/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7',
 '/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/plat-darwin',
 '/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/lib-tk',
 '/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/plat-mac',
 '/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/plat-mac/lib-scriptpackages',
 '/Users/taylorallred/Desktop/env/lib/python2.7/site-packages']
Server time:    Sat, 11 Oct 2014 07:02:34 +0000

I have looked in several places on the web, not just StackOverflow for the solution, it seems like in general that the problem is with my database and need to delete it then remake it, I have done this several times, the tutorial even has me delete the database and remake it at the point. Here is my models.py:

from django.db import models
from pygments.lexers import get_all_lexers
from pygments.styles import get_all_styles
from pygments.lexers import get_lexer_by_name
from pygments.formatters.html import HtmlFormatter
from pygments import highlight


LEXERS = [item for item in get_all_lexers() if item[1]]
LANGUAGE_CHOICES = sorted([(item[1][0], item[0]) for item in LEXERS])
STYLE_CHOICES = sorted((item, item) for item in get_all_styles())



class Snippet(models.Model):
    owner = models.ForeignKey('auth.User', related_name='snippets')
    highlighted = models.TextField()
    created = models.DateTimeField(auto_now_add=True)
    title = models.CharField(max_length=100, blank=True, default='')
    code = models.TextField()
    linenos = models.BooleanField(default=False)
    language = models.CharField(choices=LANGUAGE_CHOICES,
                                            default='python',
                                            max_length=100)
    style = models.CharField(choices=STYLE_CHOICES,
                                     default='friendly',
                                     max_length=100)
    class Meta:
        ordering = ('created',)
def save(self, *args, **kwargs):
    """
    Use the 'pygments' library to create a highlighted HTML
    representation of the code snippet.
    """
    lexer = get_lexer_by_name(self.language)
    linenos = self.linenos and 'table' or False
    options = self.title and {'title': self.title} or {}
    formatter = HtmlFormatter(style=self.style, linenos=linenos,
                                      full=true, **options)
    self.highlighted = highlight(self.code, lexer, formatter)
    super(Snippet, self).save(*args, **kwargs)

My serializers.py:

from django.forms import widgets
from rest_framework import serializers
from snippets.models import Snippet, LANGUAGE_CHOICES, STYLE_CHOICES
from django.contrib.auth.models import User



class SnippetSerializer(serializers.ModelSerializer):
    owner = serializers.Field(source='owner.username')
    class Meta:
        model = Snippet
        fields = ('id', 'title', 'code', 'linenos', 'language', 'style', 'owner')


class UserSerializer(serializers.ModelSerializer):
    snippets = serializers.PrimaryKeyRelatedField(many=True)


    class Meta:
        model = User
        fields = ('id', 'username', 'snippets')

My views.py:

from snippets.models import Snippet
from snippets.serializers import SnippetSerializer
from rest_framework import generics
from django.contrib.auth.models import User
from snippets.serializers import UserSerializer
from rest_framework import permissions

class SnippetList(generics.ListCreateAPIView):
    """
    List all snippets, or create a new snippet.
    """
    queryset = Snippet.objects.all()
    serializer_class = SnippetSerializer
    def pre_save(self, obj):
        obj.owner = self.request.user
    permission_classes = (permissions.IsAuthenticatedOrReadOnly,)

class SnippetDetail(generics.RetrieveUpdateDestroyAPIView):
    """
    Retrieve, update or delete a nippet instance.
    """
    queryset = Snippet.objects.all()
    serializer_class = SnippetSerializer
    def pre_save(self, obj):
        obj.owner = self.request.user
    permission_classes = (permissions.IsAuthenticatedOrReadOnly,)

class UserList(generics.ListAPIView):
    queryset = User.objects.all()
    serializer_class = UserSerializer

class UserDetail(generics.RetrieveAPIView):
    queryset = User.objects.all()
    serializer_class = UserSerializer

And finally my urls.py

from django.conf.urls import include
from django.conf.urls import patterns, url
from rest_framework.urlpatterns import format_suffix_patterns
from snippets import views


urlpatterns = patterns('',
    url(r'^snippets/$', views.SnippetList.as_view()),
    url(r'^snippets/(?P<pk>[0-9]+)/$', views.SnippetDetail.as_view()),
    url(r'^users/$', views.UserList.as_view()),
    url(r'^users/(?P<pk>[0-9]+)/$', views.UserDetail.as_view()),
)

urlpatterns = format_suffix_patterns(urlpatterns)

urlpatterns += patterns('',
    url(r'^api-auth/', include('rest_framework.urls',
                                       namespace='rest_framework')),
)

I apologize if I posted a bunch of unnecessary info. Thanks in advance guys.

Edit: DB Schema:

CREATE TABLE "snippets_snippet" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, 
"created" datetime NOT NULL, "title" varchar(100) NOT NULL, "code" text NOT NULL, 
"linenos" bool NOT NULL, "language" varchar(100) NOT NULL, "style" varchar(100) NOT NULL);

After doing some digging I found that when deleting and recreating the DB (as the tutorial says to) instead of using the make migrations command it would not only NOT add the columns but it would also not tell me something was wrong when running the make migrations command it tells me:

You are trying to add a non-nullable field 'highlighted' to snippet without a default;
we can't do that (the database needs something to populate existing rows).
Please select a fix:
 1) Provide a one-off default now (will be set on all existing rows)
 2) Quit, and let me add a default in models.py

If I comment out the highlighted section in models.py it will post the same message above but for the owner line. So it wants a default for both highlighted and owner, but I am not sure what to use as it. As well as the tutorial isn't helping me either on it.

Answer 1

As you went through the tutorial you must have come across the section on migration, as this was one of the major changes in Django 1.7

Prior to Django 1.7, the syncdb command never made any change that had a chance to destroy data currently in the database. This meant that if you did syncdb for a model, then added a new row to the model (a new column, effectively), syncdb would not affect that change in the database.

So either you dropped that table by hand and then ran syncdb again (to recreate it from scratch, losing any data), or you manually entered the correct statements at the database to add only that column.

Then a project came along called south which implemented migrations. This meant that there was a way to migrate forward (and reverse, undo) any changes to the database and preserve the integrity of data.

In Django 1.7, the functionality of south was integrated directly into Django. When working with migrations, the process is a bit different.

1. Make changes to models.py (as normal).
2. Create a migration. This generates code to go from the current state to the next state of your model. This is done with the makemigrations command. This command is smart enough to detect what has changed and will create a script to effect that change to your database.
3. Next, you apply that migration with migrate. This command applies all migrations in order.

So your normal syncdb is now a two-step process, python manage.py makemigrations followed by python manage.py migrate.

Now, on to your specific problem:

class Snippet(models.Model):
    owner = models.ForeignKey('auth.User', related_name='snippets')
    highlighted = models.TextField()
    created = models.DateTimeField(auto_now_add=True)
    title = models.CharField(max_length=100, blank=True, default='')
    code = models.TextField()
    linenos = models.BooleanField(default=False)
    language = models.CharField(choices=LANGUAGE_CHOICES,
                                            default='python',
                                            max_length=100)
    style = models.CharField(choices=STYLE_CHOICES,
                                     default='friendly',
                                     max_length=100)

In this model, you have two fields highlighted and code that is required (they cannot be null).

Had you added these fields from the start, there wouldn't be a problem because the table has no existing rows?

However, if the table has already been created and you add a field that cannot be null, you have to define a default value to provide for any existing rows - otherwise, the database will not accept your changes because they would violate the data integrity constraints.

This is what the command is prompting you about. You can tell Django to apply a default during migration, or you can give it a "blank" default highlighted = models.TextField(default='') in the model itself.