From c8796a385c0983ec3e0d8720efe2a18d1c360147 Mon Sep 17 00:00:00 2001 From: eric Date: Tue, 5 Mar 2013 19:14:20 -0500 Subject: [PATCH 1/5] just change endless pagination version seems to work, but need to migrate to take advantage --- requirements_versioned.pip | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_versioned.pip b/requirements_versioned.pip index a4cce5f5..2682854c 100644 --- a/requirements_versioned.pip +++ b/requirements_versioned.pip @@ -16,7 +16,7 @@ distribute==0.6.28 django-celery==3.0.9 #django-ckeditor==3.6.2.1 git+ssh://git@github.com/Gluejar/django-ckeditor.git@24350a6fba78e38682008e468741661a75483591 -django-endless-pagination==1.1 +django-endless-pagination==2.0 django-extensions==0.9 django-kombu==0.9.4 django-maintenancemode==0.10 From a44a505157895f049a052120089204ca39129a50 Mon Sep 17 00:00:00 2001 From: eric Date: Wed, 6 Mar 2013 21:54:12 -0500 Subject: [PATCH 2/5] optimize the shit out of work lists [#45623361] so, it turns out you need a migration to apply db-index changes to your db. blazing speed ensues. --- core/migrations/0041_auto.py | 250 +++++++++++++++++++++++++++++++++++ core/models.py | 2 +- frontend/views.py | 13 +- 3 files changed, 258 insertions(+), 7 deletions(-) create mode 100644 core/migrations/0041_auto.py diff --git a/core/migrations/0041_auto.py b/core/migrations/0041_auto.py new file mode 100644 index 00000000..861f6e41 --- /dev/null +++ b/core/migrations/0041_auto.py @@ -0,0 +1,250 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding index on 'Campaign', fields ['deadline'] + db.create_index('core_campaign', ['deadline']) + + # Adding index on 'Work', fields ['num_wishes'] + db.create_index('core_work', ['num_wishes']) + + # Adding index on 'Edition', fields ['publisher'] + db.create_index('core_edition', ['publisher']) + + # Adding index on 'Ebook', fields ['rights'] + db.create_index('core_ebook', ['rights']) + + + def backwards(self, orm): + # Removing index on 'Ebook', fields ['rights'] + db.delete_index('core_ebook', ['rights']) + + # Removing index on 'Edition', fields ['publisher'] + db.delete_index('core_edition', ['publisher']) + + # Removing index on 'Work', fields ['num_wishes'] + db.delete_index('core_work', ['num_wishes']) + + # Removing index on 'Campaign', fields ['deadline'] + db.delete_index('core_campaign', ['deadline']) + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'core.author': { + 'Meta': {'object_name': 'Author'}, + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'editions': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'authors'", 'symmetrical': 'False', 'to': "orm['core.Edition']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '500'}) + }, + 'core.badge': { + 'Meta': {'object_name': 'Badge'}, + 'description': ('django.db.models.fields.TextField', [], {'default': "''", 'null': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '72', 'blank': 'True'}) + }, + 'core.campaign': { + 'Meta': {'object_name': 'Campaign'}, + 'activated': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'amazon_receiver': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'deadline': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'description': ('ckeditor.fields.RichTextField', [], {'null': 'True'}), + 'details': ('ckeditor.fields.RichTextField', [], {'null': 'True', 'blank': 'True'}), + 'edition': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'campaigns'", 'null': 'True', 'to': "orm['core.Edition']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'left': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '14', 'decimal_places': '2'}), + 'license': ('django.db.models.fields.CharField', [], {'default': "'CC BY-NC-ND'", 'max_length': '255'}), + 'managers': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'campaigns'", 'symmetrical': 'False', 'to': "orm['auth.User']"}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '500', 'null': 'True'}), + 'paypal_receiver': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}), + 'status': ('django.db.models.fields.CharField', [], {'default': "'INITIALIZED'", 'max_length': '15', 'null': 'True'}), + 'target': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '14', 'decimal_places': '2'}), + 'work': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'campaigns'", 'to': "orm['core.Work']"}) + }, + 'core.campaignaction': { + 'Meta': {'object_name': 'CampaignAction'}, + 'campaign': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'actions'", 'to': "orm['core.Campaign']"}), + 'comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'timestamp': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'type': ('django.db.models.fields.CharField', [], {'max_length': '15'}) + }, + 'core.celerytask': { + 'Meta': {'object_name': 'CeleryTask'}, + 'active': ('django.db.models.fields.NullBooleanField', [], {'default': 'True', 'null': 'True', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime(2013, 3, 6, 0, 0)', 'auto_now_add': 'True', 'blank': 'True'}), + 'description': ('django.db.models.fields.CharField', [], {'max_length': '2048', 'null': 'True'}), + 'function_args': ('django.db.models.fields.IntegerField', [], {'null': 'True'}), + 'function_name': ('django.db.models.fields.CharField', [], {'max_length': '1024'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'task_id': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'tasks'", 'null': 'True', 'to': "orm['auth.User']"}) + }, + 'core.claim': { + 'Meta': {'object_name': 'Claim'}, + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'rights_holder': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'claim'", 'to': "orm['core.RightsHolder']"}), + 'status': ('django.db.models.fields.CharField', [], {'default': "'pending'", 'max_length': '7'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'claim'", 'to': "orm['auth.User']"}), + 'work': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'claim'", 'to': "orm['core.Work']"}) + }, + 'core.ebook': { + 'Meta': {'object_name': 'Ebook'}, + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'edition': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'ebooks'", 'to': "orm['core.Edition']"}), + 'format': ('django.db.models.fields.CharField', [], {'max_length': '25'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'provider': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'rights': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'db_index': 'True'}), + 'url': ('django.db.models.fields.URLField', [], {'max_length': '1024'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True'}) + }, + 'core.edition': { + 'Meta': {'object_name': 'Edition'}, + 'cover_image': ('django.db.models.fields.URLField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'public_domain': ('django.db.models.fields.NullBooleanField', [], {'null': 'True', 'blank': 'True'}), + 'publication_date': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'publisher': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '1000'}), + 'unglued': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'work': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'editions'", 'null': 'True', 'to': "orm['core.Work']"}) + }, + 'core.identifier': { + 'Meta': {'unique_together': "(('type', 'value'),)", 'object_name': 'Identifier'}, + 'edition': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'identifiers'", 'null': 'True', 'to': "orm['core.Edition']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'type': ('django.db.models.fields.CharField', [], {'max_length': '4'}), + 'value': ('django.db.models.fields.CharField', [], {'max_length': '31'}), + 'work': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'identifiers'", 'to': "orm['core.Work']"}) + }, + 'core.key': { + 'Meta': {'object_name': 'Key'}, + 'encrypted_value': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + }, + 'core.premium': { + 'Meta': {'object_name': 'Premium'}, + 'amount': ('django.db.models.fields.DecimalField', [], {'max_digits': '10', 'decimal_places': '0'}), + 'campaign': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'premiums'", 'null': 'True', 'to': "orm['core.Campaign']"}), + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'description': ('django.db.models.fields.TextField', [], {'null': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'limit': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'type': ('django.db.models.fields.CharField', [], {'max_length': '2'}) + }, + 'core.rightsholder': { + 'Meta': {'object_name': 'RightsHolder'}, + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'email': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'owner': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rights_holder'", 'to': "orm['auth.User']"}), + 'rights_holder_name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'core.subject': { + 'Meta': {'ordering': "['name']", 'object_name': 'Subject'}, + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '200'}), + 'works': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'subjects'", 'symmetrical': 'False', 'to': "orm['core.Work']"}) + }, + 'core.userprofile': { + 'Meta': {'object_name': 'UserProfile'}, + 'badges': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'holders'", 'symmetrical': 'False', 'to': "orm['core.Badge']"}), + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'facebook_id': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True'}), + 'goodreads_auth_secret': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'goodreads_auth_token': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'goodreads_user_id': ('django.db.models.fields.CharField', [], {'max_length': '32', 'null': 'True', 'blank': 'True'}), + 'goodreads_user_link': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'goodreads_user_name': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'home_url': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'librarything_id': ('django.db.models.fields.CharField', [], {'max_length': '31', 'blank': 'True'}), + 'pic_url': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}), + 'tagline': ('django.db.models.fields.CharField', [], {'max_length': '140', 'blank': 'True'}), + 'twitter_id': ('django.db.models.fields.CharField', [], {'max_length': '15', 'blank': 'True'}), + 'user': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'profile'", 'unique': 'True', 'to': "orm['auth.User']"}) + }, + 'core.waswork': { + 'Meta': {'object_name': 'WasWork'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'moved': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True'}), + 'was': ('django.db.models.fields.IntegerField', [], {'unique': 'True'}), + 'work': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['core.Work']"}) + }, + 'core.wishes': { + 'Meta': {'object_name': 'Wishes', 'db_table': "'core_wishlist_works'"}, + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'source': ('django.db.models.fields.CharField', [], {'max_length': '15', 'blank': 'True'}), + 'wishlist': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['core.Wishlist']"}), + 'work': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'wishes'", 'to': "orm['core.Work']"}) + }, + 'core.wishlist': { + 'Meta': {'object_name': 'Wishlist'}, + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'user': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'wishlist'", 'unique': 'True', 'to': "orm['auth.User']"}), + 'works': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'wishlists'", 'symmetrical': 'False', 'through': "orm['core.Wishes']", 'to': "orm['core.Work']"}) + }, + 'core.work': { + 'Meta': {'ordering': "['title']", 'object_name': 'Work'}, + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'description': ('django.db.models.fields.TextField', [], {'default': "''", 'null': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'language': ('django.db.models.fields.CharField', [], {'default': "'en'", 'max_length': '2'}), + 'num_wishes': ('django.db.models.fields.IntegerField', [], {'default': '0', 'db_index': 'True'}), + 'openlibrary_lookup': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '1000'}) + } + } + + complete_apps = ['core'] \ No newline at end of file diff --git a/core/models.py b/core/models.py index 8be6bbee..d2fc1b56 100755 --- a/core/models.py +++ b/core/models.py @@ -832,7 +832,7 @@ class Subject(models.Model): class Edition(models.Model): created = models.DateTimeField(auto_now_add=True) title = models.CharField(max_length=1000) - publisher = models.CharField(max_length=255, null=True, blank=True) + publisher = models.CharField(max_length=255, null=True, blank=True, db_index=True) publication_date = models.CharField(max_length=50, null=True, blank=True) public_domain = models.NullBooleanField(null=True, blank=True) work = models.ForeignKey("Work", related_name="editions", null=True) diff --git a/frontend/views.py b/frontend/views.py index 699b0d41..7970cc7f 100755 --- a/frontend/views.py +++ b/frontend/views.py @@ -484,16 +484,17 @@ recommended_user = User.objects.filter( username=settings.UNGLUEIT_RECOMMENDED_U class WorkListView(FilterableListView): template_name = "work_list.html" context_object_name = "work_list" - max_works=20 + max_works=100000 def get_queryset_all(self): facet = self.kwargs['facet'] if (facet == 'popular'): - return models.Work.objects.order_by('-num_wishes', 'id') + return models.Work.objects.exclude(num_wishes=0).order_by('-num_wishes', 'id') elif (facet == 'recommended'): + self.template_name = "recommended.html" return models.Work.objects.filter(wishlists__user=recommended_user).order_by('-num_wishes') elif (facet == 'new'): - return models.Work.objects.filter(num_wishes__gt=0).order_by('-created', '-num_wishes' ,'id') + return models.Work.objects.exclude(num_wishes=0).order_by('-created', '-num_wishes' ,'id') else: return models.Work.objects.all().order_by('-created', 'id') @@ -502,7 +503,7 @@ class WorkListView(FilterableListView): qs=self.get_queryset() context['ungluers'] = userlists.work_list_users(qs,5) context['facet'] = self.kwargs.get('facet','') - works_unglued = qs.filter(editions__ebooks__isnull=False).distinct() | qs.filter(campaigns__status='SUCCESSFUL').distinct() + works_unglued = qs.exclude(editions__ebooks__isnull=True).distinct() | qs.filter(campaigns__status='SUCCESSFUL').distinct() context['works_unglued'] = works_unglued.order_by('-campaigns__status', 'campaigns__deadline', '-num_wishes')[:self.max_works] context['works_active'] = qs.filter(campaigns__status='ACTIVE').distinct()[:self.max_works] context['works_wished'] = qs.exclude(editions__ebooks__isnull=False).exclude(campaigns__status='ACTIVE').exclude(campaigns__status='SUCCESSFUL').distinct()[:self.max_works] @@ -520,7 +521,7 @@ class WorkListView(FilterableListView): class ByPubListView(WorkListView): template_name = "bypub_list.html" context_object_name = "work_list" - max_works=100 + max_works=100000 def get_queryset_all(self): facet = self.kwargs.get('facet','') @@ -529,7 +530,7 @@ class ByPubListView(WorkListView): if (facet == 'popular'): return objects.order_by('-num_wishes', 'id') elif (facet == 'pubdate'): - return objects.order_by('-editions__publication_date') + return objects.order_by('-editions__publication_date') # turns out this messes up distinct, and MySQL doesn't support DISTINCT ON elif (facet == 'new'): return objects.filter(num_wishes__gt=0).order_by('-created', '-num_wishes' ,'id') else: From 3ccad0770bb20b841aff3f426fc1d77355b62412 Mon Sep 17 00:00:00 2001 From: eric Date: Wed, 6 Mar 2013 21:54:48 -0500 Subject: [PATCH 3/5] refactor work list templates --- frontend/templates/bypub_list.html | 136 +--------------------------- frontend/templates/recommended.html | 22 +++++ frontend/templates/work_list.html | 42 ++------- 3 files changed, 34 insertions(+), 166 deletions(-) create mode 100644 frontend/templates/recommended.html diff --git a/frontend/templates/bypub_list.html b/frontend/templates/bypub_list.html index 27b88734..095eca00 100644 --- a/frontend/templates/bypub_list.html +++ b/frontend/templates/bypub_list.html @@ -1,138 +1,8 @@ -{% extends "base.html" %} +{% extends "work_list.html" %} {% load endless %} {% load lang_utils %} {% block title %} Works published by {{ pubname }} {% endblock %} -{% block extra_css %} - - - - -{% endblock %} -{% block extra_head %} - - - - - - - -{% endblock %} -{% block topsection %} -
{{ activetab }}
-
-
-
-
-
-
-
-
-
{{ facet|capfirst }}
-
-
Books from {{ pubname }} {% if pub_lang %}( {{pub_lang|ez_lang_name}} ) {% endif %} -
-
-
-
-
-
-
-
- -
-
- -{% endblock %} -{% block content %} -
-
-
- {% include "explore.html" %} -
- -
-
-
-
- - -
-
- {% ifequal work_list.count 0 %} - There aren't any {{ pub_lang|ez_lang_name }} works in this list yet. Why not add your favorite books to your wishlist, so we can feature them here? - {% else %} - {% lazy_paginate 20 works_unglued using "works_unglued" %} - {% for work in works_unglued %} -
- {% with work.last_campaign_status as status %} - {% with work.last_campaign.deadline as deadline %} - {% with work.googlebooks_id as googlebooks_id %} - {% include "book_panel.html" %} - {% endwith %}{% endwith %}{% endwith %} -
- {% endfor %} - - - {% lazy_paginate 20 works_active using "works_active" %} - {% for work in works_active %} -
- {% with work.last_campaign_status as status %} - {% with work.last_campaign.deadline as deadline %} - {% with work.googlebooks_id as googlebooks_id %} - {% include "book_panel.html" %} - {% endwith %}{% endwith %}{% endwith %} -
- {% endfor %} - - - {% lazy_paginate 20 works_wished using "works_wished" %} - {% for work in works_wished %} -
- {% with work.last_campaign_status as status %} - {% with work.last_campaign.deadline as deadline %} - {% with work.googlebooks_id as googlebooks_id %} - {% include "book_panel.html" %} - {% endwith %}{% endwith %}{% endwith %} -
- {% endfor %} - - {% endifequal %} -
-
-
-
-
-
+{% block userblock2 %} +Books from {{ pubname }} {% if pub_lang %}( {{pub_lang|ez_lang_name}} ) {% endif %} {% endblock %} diff --git a/frontend/templates/recommended.html b/frontend/templates/recommended.html new file mode 100644 index 00000000..8c2cbbc1 --- /dev/null +++ b/frontend/templates/recommended.html @@ -0,0 +1,22 @@ +{% extends "work_list.html" %} +{% load endless %} +{% load lang_utils %} + +{% block title %} Works published by {{ pubname }} {% endblock %} +{% block noworks %} +Check back soon to see what we're recommending. +{% endblock %} + +{% block userblock %} +
+
Staff Picks
+
+
Here are the {% if pub_lang %}{{pub_lang|ez_lang_name}} language {% endif %}books Amanda, Andromeda, Eric, and Raymond are loving lately. +
+
+ Picture of Amanda + Picture of Andromeda + Picture of Eric + Picture of Raymond +
+{% endblock %} diff --git a/frontend/templates/work_list.html b/frontend/templates/work_list.html index f0ae7494..d0b13f81 100644 --- a/frontend/templates/work_list.html +++ b/frontend/templates/work_list.html @@ -27,26 +27,18 @@
-
- {% if facet == 'recommended' %} -
Staff Picks
-
-
Here are the {% if pub_lang %}{{pub_lang|ez_lang_name}} language {% endif %}books Amanda, Andromeda, Eric, and Raymond are loving lately. -
-
- Picture of Amanda - Picture of Andromeda - Picture of Eric - Picture of Raymond -
- {% else %} + {% block userblock %} +
{{ facet|capfirst }}
-
With your help we're raising money to buy the rights to give these {% if pub_lang %}{{pub_lang|ez_lang_name}} language {% endif %}books to the world. +
+ {% block userblock2 %} + With your help we're raising money to buy the rights to give these {% if pub_lang %}{{pub_lang|ez_lang_name}} language {% endif %}books to the world. + {% endblock %}
- {% endif %} + {% endblock %}
@@ -90,16 +82,11 @@
{% ifequal work_list.count 0 %} - {% if facet == 'recommended' %} - Check back soon to see what we're recommending. - {% else %} + {% block noworks %} There aren't any {{ pub_lang|ez_lang_name }} works in this list yet. Why not add your favorite books to your wishlist, so we can feature them here? - {% endif %} + {% endblock %} {% else %} - {% comment %} - pagination commented out for now because it was causing huge db hits/pageload times. These sets have been limited in views.py to 20 on the theory that people only want to see the MOST popular works, and the other facets are unlikely to produce sets exceeding this anyway. {% lazy_paginate 20 works_unglued using "works_unglued" %} - {% endcomment %} {% for work in works_unglued %}
{% with work.last_campaign_status as status %} @@ -109,18 +96,13 @@ {% endwith %}{% endwith %}{% endwith %}
{% endfor %} - {% comment %} - {% endcomment %} - - {% comment %} {% lazy_paginate 20 works_active using "works_active" %} - {% endcomment %} {% for work in works_active %}
{% with work.last_campaign_status as status %} @@ -130,18 +112,14 @@ {% endwith %}{% endwith %}{% endwith %}
{% endfor %} - {% comment %} - {% endcomment %} - {% comment %} {% lazy_paginate 20 works_wished using "works_wished" %} - {% endcomment %} {% for work in works_wished %}
{% with work.last_campaign_status as status %} @@ -151,14 +129,12 @@ {% endwith %}{% endwith %}{% endwith %}
{% endfor %} - {% comment %} - {% endcomment %} {% endifequal %}
From 05d37163eaa1a37539cf3e58ea523286ca2405b3 Mon Sep 17 00:00:00 2001 From: eric Date: Thu, 7 Mar 2013 11:37:40 -0500 Subject: [PATCH 4/5] despam descriptions in db wasn't real happy with this: https://unglue.it/work/66938/ so I looked for other spammy descriptions in the db --- core/bookloader.py | 13 +++++++++++++ core/management/commands/despam_descriptions.py | 17 +++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 core/management/commands/despam_descriptions.py diff --git a/core/bookloader.py b/core/bookloader.py index cbccb436..a14e224c 100755 --- a/core/bookloader.py +++ b/core/bookloader.py @@ -488,6 +488,17 @@ def merge_works(w1, w2, user=None): w2.delete() +def despam_description(description): + """ a lot of descriptions from openlibrary have free-book promotion text; this removes some of it.""" + if description.find("GeneralBooksClub.com")>-1 or description.find("AkashaPublishing.Com")>-1: + return "" + pieces=description.split("1stWorldLibrary.ORG -") + if len(pieces)>1: + return pieces[1] + pieces=description.split("a million books for free.") + if len(pieces)>1: + return pieces[1] + return description def add_openlibrary(work, hard_refresh = False): if (not hard_refresh) and work.openlibrary_lookup is not None: @@ -541,6 +552,7 @@ def add_openlibrary(work, hard_refresh = False): if isinstance(description,dict): if description.has_key('value'): description=description['value'] + description=despam_description(description) if not work.description or work.description.startswith('{') or len(description) > len(work.description): work.description = description work.save() @@ -710,3 +722,4 @@ def add_missing_isbn_to_editions(max_num=None, confirm=False): class LookupFailure(Exception): pass + diff --git a/core/management/commands/despam_descriptions.py b/core/management/commands/despam_descriptions.py new file mode 100644 index 00000000..7faa3cc0 --- /dev/null +++ b/core/management/commands/despam_descriptions.py @@ -0,0 +1,17 @@ + +from django.core.management.base import BaseCommand +from regluit.core import models, bookloader + +class Command(BaseCommand): + help = "check description db for free ebook spam" + + def handle(self, **options): + spam_strings=["1stWorldLibrary.ORG", "GeneralBooksClub.com", "million-books.com", "AkashaPublishing.Com"] + for spam_string in spam_strings: + qs=models.Work.objects.filter(description__icontains=spam_string) + print "Number of Works with %s in description: %s" % (spam_string, qs.count()) + + for work in qs: + work.description = bookloader.despam_description(work.description) + print "updating work %s" % work + bookloader.add_openlibrary(work, hard_refresh = True) From 99ba3499a74f9aa2cf2d9b600eed5a81fbf0ad51 Mon Sep 17 00:00:00 2001 From: thatandromeda Date: Tue, 12 Mar 2013 11:58:33 -0300 Subject: [PATCH 5/5] Update signals.py --- core/signals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/signals.py b/core/signals.py index 5a0254bd..71930f24 100644 --- a/core/signals.py +++ b/core/signals.py @@ -91,7 +91,7 @@ def create_notice_types(app, created_models, verbosity, **kwargs): notification.create_notice_type("wishlist_successful", _("Successful Campaign"), _("An ungluing campaign that you have supported or followed has succeeded.")) notification.create_notice_type("wishlist_unsuccessful", _("Unsuccessful Campaign"), _("An ungluing campaign that you supported didn't succeed this time.")) notification.create_notice_type("wishlist_updated", _("Campaign Updated"), _("An ungluing campaign you support has been updated."), default = 1) - notification.create_notice_type("wishlist_message", _("Campaign Communication"), _("There's a message from unglue.it staff or the rights holder about a campaign on your wishlist.")) + notification.create_notice_type("wishlist_message", _("Campaign Communication"), _("You have a private message from unglue.it staff or the rights holder about a campaign on your wishlist.")) notification.create_notice_type("wishlist_price_drop", _("Campaign Price Drop"), _("An ungluing campaign you're interested in has a reduced target."), default = 1) notification.create_notice_type("wishlist_unglued_book_released", _("Unglued Book!"), _("A book you wanted is now available to be downloaded.")) notification.create_notice_type("pledge_you_have_pledged", _("Thanks For Your Pledge!"), _("Your ungluing pledge has been entered."))