464 lines
12 KiB
Plaintext
464 lines
12 KiB
Plaintext
{
|
|
"metadata": {
|
|
"name": "",
|
|
"signature": "sha256:2a1e6efa2c9f22f584f68cb52a282e34820d954023d8a8860c1f96979e2b4697"
|
|
},
|
|
"nbformat": 3,
|
|
"nbformat_minor": 0,
|
|
"worksheets": [
|
|
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"collapsed": false,
|
|
"input": [
|
|
"from itertools import islice\n",
|
|
"\n",
|
|
"from regluit.core.models import Campaign\n",
|
|
"from regluit.core.mobigen import (edition_mobi_status,\n",
|
|
" editions_to_convert,\n",
|
|
" convert_to_mobi,\n",
|
|
" generate_mobi_ebook_for_edition)"
|
|
],
|
|
"language": "python",
|
|
"metadata": {},
|
|
"outputs": []
|
|
},
|
|
{
|
|
"cell_type": "heading",
|
|
"level": 1,
|
|
"metadata": {},
|
|
"source": [
|
|
"Make sure the Campaigns are of the correct type given the licensing of kindlegen"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"**I think T4U and pledge campaigns but not B2U.**"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"collapsed": false,
|
|
"input": [
|
|
"from regluit.core.models import Campaign\n",
|
|
"from regluit.core import parameters\n",
|
|
"(parameters.REWARDS, parameters.BUY2UNGLUE, parameters.THANKS)"
|
|
],
|
|
"language": "python",
|
|
"metadata": {},
|
|
"outputs": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"collapsed": false,
|
|
"input": [
|
|
"list(editions_to_convert())"
|
|
],
|
|
"language": "python",
|
|
"metadata": {},
|
|
"outputs": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"collapsed": false,
|
|
"input": [
|
|
"e = list(editions_to_convert())[0]\n",
|
|
"e.work.last_campaign().type"
|
|
],
|
|
"language": "python",
|
|
"metadata": {},
|
|
"outputs": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"collapsed": false,
|
|
"input": [
|
|
"[edition.work.last_campaign().type for edition in editions_to_convert()]"
|
|
],
|
|
"language": "python",
|
|
"metadata": {},
|
|
"outputs": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"collapsed": false,
|
|
"input": [
|
|
"# http://127.0.0.1:8000/work/138133/ --> The Global Librarian\n",
|
|
"\n",
|
|
"edition = list(islice(editions_to_convert(),1))[0]\n",
|
|
"edition.work.ebooks(), edition.work.id"
|
|
],
|
|
"language": "python",
|
|
"metadata": {},
|
|
"outputs": []
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"the following is test invocation of the mobigen API"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"collapsed": false,
|
|
"input": [
|
|
"%%bash\n",
|
|
"\n",
|
|
"curl -k --user \"admin:CXq5FSEQFgXtP_s\" -X POST -H \"Content-Type: application/epub+zip\" --data-binary \"@/Users/raymondyee/D/Document/Gluejar/Gluejar.github/regluit/test-data/pg2701.epub\" https://docker.gluejar.com:5001/mobigen > pg2701.mobi"
|
|
],
|
|
"language": "python",
|
|
"metadata": {},
|
|
"outputs": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"collapsed": false,
|
|
"input": [
|
|
"# https://archive.org/download/mobydickorthewha02701gut/pg2701.epub\n",
|
|
"# \"https://archive.org/download/Feeding_the_City/9781909254039_Feeding_the_City.epub\" --> for a big epub\n",
|
|
"\n",
|
|
"from regluit.core.mobigen import convert_to_mobi\n",
|
|
"\n",
|
|
"output = convert_to_mobi(\"https://archive.org/download/mobydickorthewha02701gut/pg2701.epub\")\n",
|
|
"\n",
|
|
"with open(\"/Users/raymondyee/Downloads/pg2701.mobi\", \"wb\") as f:\n",
|
|
" f.write(output)\n"
|
|
],
|
|
"language": "python",
|
|
"metadata": {},
|
|
"outputs": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"collapsed": false,
|
|
"input": [
|
|
"# how to work with EbookFile\n",
|
|
"# https://github.com/Gluejar/regluit/blob/792659c325a7bee2b49337408336fdeadab3464a/core/models.py#L904\n",
|
|
"# Campaign."
|
|
],
|
|
"language": "python",
|
|
"metadata": {},
|
|
"outputs": []
|
|
},
|
|
{
|
|
"cell_type": "heading",
|
|
"level": 1,
|
|
"metadata": {},
|
|
"source": [
|
|
"Using the low level Django file storage API: File Storage API"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"I need to remind myself about how the [File Storage API](https://docs.djangoproject.com/en/1.4/ref/files/storage/) works.\n",
|
|
"\n",
|
|
"We can use `default_storage` directly to read, write [file storage objects](https://docs.djangoproject.com/en/1.4/topics/files/#storage-objects) and to test for existence.\n",
|
|
"\n",
|
|
"I'm a bit unclear about the relevance of [FileField](https://docs.djangoproject.com/en/1.4/ref/models/fields/#filefield).\n",
|
|
"\n",
|
|
"And how does Ebookfile work? Look at https://github.com/Gluejar/regluit/blob/f7b796c6a6d220f6475dbfdc0a8aeb16a09e84b1/core/models.py#L1777:\n",
|
|
"\n",
|
|
"```python\n",
|
|
"class EbookFile(models.Model):\n",
|
|
" file = models.FileField(upload_to=path_for_file)\n",
|
|
"``` \n",
|
|
"\n",
|
|
"I should be able to find hints about how to instantiate an `EbookFile` in the right way."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"BTW, is there a tension between the standard API and extras that are being used by S3storages https://django-storages.readthedocs.org/en/latest/backends/amazon-S3.html?"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"collapsed": false,
|
|
"input": [
|
|
"from django.core.files.storage import default_storage\n",
|
|
"from django.core.files.base import ContentFile, File"
|
|
],
|
|
"language": "python",
|
|
"metadata": {},
|
|
"outputs": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"collapsed": false,
|
|
"input": [
|
|
"default_storage.listdir(\"/\")"
|
|
],
|
|
"language": "python",
|
|
"metadata": {},
|
|
"outputs": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"collapsed": false,
|
|
"input": [
|
|
"# getting key pairs\n",
|
|
"default_storage.bucket.get_all_keys()"
|
|
],
|
|
"language": "python",
|
|
"metadata": {},
|
|
"outputs": []
|
|
},
|
|
{
|
|
"cell_type": "heading",
|
|
"level": 1,
|
|
"metadata": {},
|
|
"source": [
|
|
"Sample code for using default_storage"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"collapsed": false,
|
|
"input": [
|
|
"default_storage.exists('storage_test')"
|
|
],
|
|
"language": "python",
|
|
"metadata": {},
|
|
"outputs": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"collapsed": false,
|
|
"input": [
|
|
"file = default_storage.open('storage_test', 'w')\n",
|
|
"file.write('storage contents')\n",
|
|
"file.close()\n",
|
|
"\n",
|
|
"default_storage.exists('storage_test')"
|
|
],
|
|
"language": "python",
|
|
"metadata": {},
|
|
"outputs": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"collapsed": false,
|
|
"input": [
|
|
"file = default_storage.open('storage_test', 'r')\n",
|
|
"file.read()\n",
|
|
"file.close()"
|
|
],
|
|
"language": "python",
|
|
"metadata": {},
|
|
"outputs": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"collapsed": false,
|
|
"input": [
|
|
"default_storage.delete('storage_test')\n",
|
|
"default_storage.exists('storage_test')"
|
|
],
|
|
"language": "python",
|
|
"metadata": {},
|
|
"outputs": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"collapsed": false,
|
|
"input": [
|
|
"# clean up some files \n",
|
|
"\n",
|
|
"print (default_storage.bucket)\n",
|
|
"\n",
|
|
"for key in default_storage.listdir(\"/ebf\")[1]:\n",
|
|
" print default_storage.delete(\"/ebf/\" + key)\n"
|
|
],
|
|
"language": "python",
|
|
"metadata": {},
|
|
"outputs": []
|
|
},
|
|
{
|
|
"cell_type": "heading",
|
|
"level": 1,
|
|
"metadata": {},
|
|
"source": [
|
|
"How to write the results of the conversion"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"collapsed": false,
|
|
"input": [
|
|
"from regluit.core.models import EbookFile\n",
|
|
"from django.core.files.storage import default_storage\n",
|
|
"from django.core.files.base import ContentFile, File\n",
|
|
"\n",
|
|
"# http://stackoverflow.com/a/519653\n",
|
|
"\n",
|
|
"def read_in_chunks(file_object, chunk_size=1024):\n",
|
|
" \"\"\"Lazy function (generator) to read a file piece by piece.\n",
|
|
" Default chunk size: 1k.\"\"\"\n",
|
|
" while True:\n",
|
|
" data = file_object.read(chunk_size)\n",
|
|
" if not data:\n",
|
|
" break\n",
|
|
" yield data\n",
|
|
" \n",
|
|
"def write_file_to_storage(file_object, content_type, path):\n",
|
|
" \"\"\"\n",
|
|
" write file_object to the default_storage at given path\n",
|
|
" \"\"\"\n",
|
|
" file_s3 = ContentFile(file_object)\n",
|
|
" file_s3.content_type = content_type\n",
|
|
" \n",
|
|
" default_storage.save(path, file_s3)\n",
|
|
" return file_s3"
|
|
],
|
|
"language": "python",
|
|
"metadata": {},
|
|
"outputs": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"collapsed": false,
|
|
"input": [
|
|
"import uuid\n",
|
|
"\n",
|
|
"file_ = write_file_to_storage(open(\"/Users/raymondyee/Downloads/hello.mobi\").read(), \n",
|
|
" \"application/x-mobipocket-ebook\", \n",
|
|
" \"/ebf/{0}.mobi\".format(uuid.uuid4().get_hex()))\n",
|
|
"\n"
|
|
],
|
|
"language": "python",
|
|
"metadata": {},
|
|
"outputs": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"collapsed": false,
|
|
"input": [
|
|
"file_.name"
|
|
],
|
|
"language": "python",
|
|
"metadata": {},
|
|
"outputs": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"collapsed": false,
|
|
"input": [
|
|
"default_storage.url(file_.name)"
|
|
],
|
|
"language": "python",
|
|
"metadata": {},
|
|
"outputs": []
|
|
},
|
|
{
|
|
"cell_type": "heading",
|
|
"level": 1,
|
|
"metadata": {},
|
|
"source": [
|
|
"Sample: write a sample mobi file as an Ebook for a campaign book"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"collapsed": false,
|
|
"input": [
|
|
"for campaign in Campaign.objects.filter(edition__ebooks__isnull=False).distinct():\n",
|
|
" #print (campaign.edition.title, edition_mobi_status(campaign.edition))\n",
|
|
" if edition_mobi_status(campaign.edition) == 0: # possible to generate mobi\n",
|
|
" print(campaign.edition.title, campaign.edition.ebooks.filter(format='epub')[0].url)"
|
|
],
|
|
"language": "python",
|
|
"metadata": {},
|
|
"outputs": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"collapsed": false,
|
|
"input": [
|
|
"list(editions_to_convert())"
|
|
],
|
|
"language": "python",
|
|
"metadata": {},
|
|
"outputs": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"collapsed": false,
|
|
"input": [
|
|
"from itertools import islice\n",
|
|
"from StringIO import StringIO\n",
|
|
"\n",
|
|
"from regluit.core.mobigen import convert_to_mobi\n",
|
|
"\n",
|
|
"import uuid\n"
|
|
],
|
|
"language": "python",
|
|
"metadata": {},
|
|
"outputs": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"collapsed": false,
|
|
"input": [
|
|
"for edition in islice(editions_to_convert(),1):\n",
|
|
" ebook = generate_mobi_ebook_for_edition(edition)"
|
|
],
|
|
"language": "python",
|
|
"metadata": {},
|
|
"outputs": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"collapsed": false,
|
|
"input": [
|
|
"edition_mobi_status(edition)"
|
|
],
|
|
"language": "python",
|
|
"metadata": {},
|
|
"outputs": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"collapsed": false,
|
|
"input": [
|
|
"# check connection between edition and Ebook, Ebookfile before creating mobi.\n",
|
|
"[(ebook.id, edition.ebooks.all(), edition.ebook_files.all()) for ebook in edition.ebooks.all()]"
|
|
],
|
|
"language": "python",
|
|
"metadata": {},
|
|
"outputs": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"collapsed": false,
|
|
"input": [
|
|
"[(eb.url, eb.format, eb.provider, eb.rights, eb.edition) for eb in edition.ebooks.all()]"
|
|
],
|
|
"language": "python",
|
|
"metadata": {},
|
|
"outputs": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"collapsed": false,
|
|
"input": [
|
|
"# checking on relationship between Ebookfile and Ebook\n",
|
|
"\n",
|
|
"[ebf for ebf in EbookFile.objects.all() if ebf.active]"
|
|
],
|
|
"language": "python",
|
|
"metadata": {},
|
|
"outputs": []
|
|
}
|
|
],
|
|
"metadata": {}
|
|
}
|
|
]
|
|
} |