From 59d9ed4a17f4f50c0a5bc83859a32091b070fc4c Mon Sep 17 00:00:00 2001 From: Raymond Yee Date: Fri, 24 Jun 2016 14:23:03 -0700 Subject: [PATCH] change the various crontabs add notebooks/fix_build_3550.ipynb, which is not directly related to celery --- deploy/crontab_just.txt | 2 +- deploy/crontab_localvm.txt | 2 +- deploy/crontab_prod.txt | 3 +- notebooks/fix_build_3550.ipynb | 1088 ++++++-------------------------- vagrant/Vagrantfile | 2 +- 5 files changed, 185 insertions(+), 912 deletions(-) diff --git a/deploy/crontab_just.txt b/deploy/crontab_just.txt index 5a01ba40..4b8386fe 100644 --- a/deploy/crontab_just.txt +++ b/deploy/crontab_just.txt @@ -21,4 +21,4 @@ # # m h dom mon dow command * * * * * cd /opt/regluit; . /opt/regluit/ENV/bin/activate; /opt/regluit/ENV/bin/django-admin.py emit_notices --settings=regluit.settings.just > /opt/regluit/deploy/emit_notices.log 2>&1 ; touch /opt/regluit/deploy/last-cron -@reboot sudo mkdir /var/run/celery; sudo chown celery:celery /var/log/celery /var/run/celery; sudo /etc/init.d/celeryd restart; sudo /etc/init.d/celerybeat restart; +@reboot sudo mkdir /var/run/celery; sudo chown celery:celery /var/log/celery /var/run/celery; cd /opt/regluit; . /opt/regluit/ENV/bin/activate; /opt/regluit/ENV/bin/django-admin.py celeryd_multi restart w1 --settings=regluit.settings.just; /etc/init.d/celerybeat restart; diff --git a/deploy/crontab_localvm.txt b/deploy/crontab_localvm.txt index fdf0b75e..ef35f2a1 100644 --- a/deploy/crontab_localvm.txt +++ b/deploy/crontab_localvm.txt @@ -21,4 +21,4 @@ # # m h dom mon dow command * * * * * cd /opt/regluit; . /opt/regluit/ENV/bin/activate; /opt/regluit/ENV/bin/django-admin.py emit_notices --settings=regluit.settings.localvm > /opt/regluit/deploy/emit_notices.log 2>&1 ; touch /opt/regluit/deploy/last-cron -@reboot sudo mkdir /var/run/celery; sudo chown celery:celery /var/log/celery /var/run/celery; sudo /etc/init.d/celeryd restart; sudo /etc/init.d/celerybeat restart; +@reboot sudo mkdir /var/run/celery; sudo chown celery:celery /var/log/celery /var/run/celery; cd /opt/regluit; . /opt/regluit/ENV/bin/activate; /opt/regluit/ENV/bin/django-admin.py celeryd_multi restart w1 --settings=regluit.settings.localvm; /etc/init.d/celerybeat restart; \ No newline at end of file diff --git a/deploy/crontab_prod.txt b/deploy/crontab_prod.txt index b622d6f3..ca416808 100644 --- a/deploy/crontab_prod.txt +++ b/deploy/crontab_prod.txt @@ -21,5 +21,4 @@ # # m h dom mon dow command * * * * * /opt/regluit/deploy/emit_notices.sh -@reboot sudo mkdir /var/run/celery; sudo chown celery:celery /var/log/celery /var/run/celery; sudo /etc/init.d/celeryd restart; sudo /etc/init.d/celerybeat restart; - +@reboot sudo mkdir /var/run/celery; sudo chown celery:celery /var/log/celery /var/run/celery; cd /opt/regluit; . /opt/regluit/ENV/bin/activate; /opt/regluit/ENV/bin/django-admin.py celeryd_multi restart w1 --settings=regluit.settings.prod; /etc/init.d/celerybeat restart; diff --git a/notebooks/fix_build_3550.ipynb b/notebooks/fix_build_3550.ipynb index d067f209..393898c2 100644 --- a/notebooks/fix_build_3550.ipynb +++ b/notebooks/fix_build_3550.ipynb @@ -1,910 +1,184 @@ { - "metadata": { - "name": "", - "signature": "sha256:8a6db9b1acf17257f64b272a7934bc5183d130e6a0f94dde397e28c48ceab183" - }, - "nbformat": 3, - "nbformat_minor": 0, - "worksheets": [ + "cells": [ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "http://jenkins.unglueit.com/job/regluit/3550/console\n", - "\n", - "looking for book to replace the problematic one" - ] - }, - { - "cell_type": "code", - "collapsed": false, - "input": [ - "from regluit.experimental import bookdata\n", - "from regluit.core import isbn" - ], - "language": "python", - "metadata": {}, - "outputs": [], - "prompt_number": 1 - }, - { - "cell_type": "code", - "collapsed": false, - "input": [ - "from regluit.core import bookloader\n", - "\n", - "#edition = bookloader.add_by_isbn('9789571349268')\n", - "#assert edition.work.language == 'zh-TW'\n", - "\n", - "edition = bookloader.add_by_isbn('9787544706919')\n", - "assert edition.work.language == 'zh-CN'" - ], - "language": "python", - "metadata": {}, - "outputs": [], - "prompt_number": 2 - }, - { - "cell_type": "code", - "collapsed": false, - "input": [ - "r = bookloader.get_google_isbn_results('9787544706919')\n", - "r" - ], - "language": "python", - "metadata": {}, - "outputs": [ - { - "metadata": {}, - "output_type": "pyout", - "prompt_number": 3, - "text": [ - "{u'items': [{u'accessInfo': {u'accessViewStatus': u'NONE',\n", - " u'country': u'US',\n", - " u'embeddable': False,\n", - " u'epub': {u'isAvailable': False},\n", - " u'pdf': {u'isAvailable': False},\n", - " u'publicDomain': False,\n", - " u'quoteSharingAllowed': False,\n", - " u'textToSpeechPermission': u'ALLOWED',\n", - " u'viewability': u'NO_PAGES',\n", - " u'webReaderLink': u'http://books.google.com/books/reader?id=p_vaPgAACAAJ&hl=&printsec=frontcover&output=reader&source=gbs_api'},\n", - " u'etag': u'ua9KgTFG/uE',\n", - " u'id': u'p_vaPgAACAAJ',\n", - " u'kind': u'books#volume',\n", - " u'saleInfo': {u'country': u'US',\n", - " u'isEbook': False,\n", - " u'saleability': u'NOT_FOR_SALE'},\n", - " u'searchInfo': {u'textSnippet': u'\\u672c\\u4e66\\u5206\\u4e09\\u90e8\\u5206,\\u7b2c\\u4e00\\u90e8\\u300a\\u6211\\u7684\\u8eab\\u4e16\\u300b\\u8bb2\\u8ff0\\u4e86\\u5965\\u5df4\\u9a6c\\u7684\\u5bb6\\u5ead\\u80cc\\u666f\\u548c\\u4ed6\\u8f9b\\u9178\\u7684\\u7ae5\\u5e74\\u751f\\u6d3b;\\u7b2c\\u4e8c\\u90e8\\u300a\\u829d\\u52a0\\u54e5\\u8d77\\u98de\\u300b\\u63cf\\u8ff0\\u5965\\u5df4\\u9a6c\\u5728\\u829d\\u52a0\\u54e5\\u5357\\u533a\\u4e09\\u5e74\\u7684\\u793e\\u533a\\u5de5\\u4f5c ...'},\n", - " u'selfLink': u'https://www.googleapis.com/books/v1/volumes/p_vaPgAACAAJ',\n", - " u'volumeInfo': {u'allowAnonLogging': False,\n", - " u'authors': [u'\\u5965\\u5df4\\u9a6c'],\n", - " u'canonicalVolumeLink': u'http://books.google.com/books/about/%E5%A5%A5%E5%B7%B4%E9%A9%AC%E5%9B%9E%E5%BF%86%E5%BD%95.html?hl=&id=p_vaPgAACAAJ',\n", - " u'categories': [u'African Americans'],\n", - " u'contentVersion': u'preview-1.0.0',\n", - " u'description': u'\\u672c\\u4e66\\u5206\\u4e09\\u90e8\\u5206,\\u7b2c\\u4e00\\u90e8\\u300a\\u6211\\u7684\\u8eab\\u4e16\\u300b\\u8bb2\\u8ff0\\u4e86\\u5965\\u5df4\\u9a6c\\u7684\\u5bb6\\u5ead\\u80cc\\u666f\\u548c\\u4ed6\\u8f9b\\u9178\\u7684\\u7ae5\\u5e74\\u751f\\u6d3b;\\u7b2c\\u4e8c\\u90e8\\u300a\\u829d\\u52a0\\u54e5\\u8d77\\u98de\\u300b\\u63cf\\u8ff0\\u5965\\u5df4\\u9a6c\\u5728\\u829d\\u52a0\\u54e5\\u5357\\u533a\\u4e09\\u5e74\\u7684\\u793e\\u533a\\u5de5\\u4f5c;\\u7b2c\\u4e09\\u90e8\\u300a\\u68a6\\u56de\\u80af\\u5c3c\\u4e9a\\u300b\\u63cf\\u8ff0\\u5965\\u5df4\\u9a6c\\u4e0e\\u540c\\u7236\\u5f02\\u6bcd\\u7684\\u59b9\\u59b9\\u5728\\u7ebd\\u7ea6\\u89c1\\u9762\\u540e\\u5c55\\u5f00\\u7684\\u80af\\u5c3c\\u4e9a\\u90e8\\u843d\\u5bfb\\u6839\\u4e4b\\u65c5\\u3002',\n", - " u'imageLinks': {u'smallThumbnail': u'http://books.google.com/books/content?id=p_vaPgAACAAJ&printsec=frontcover&img=1&zoom=5&source=gbs_api',\n", - " u'thumbnail': u'http://books.google.com/books/content?id=p_vaPgAACAAJ&printsec=frontcover&img=1&zoom=1&source=gbs_api'},\n", - " u'industryIdentifiers': [{u'identifier': u'7544706915',\n", - " u'type': u'ISBN_10'},\n", - " {u'identifier': u'9787544706919', u'type': u'ISBN_13'}],\n", - " u'infoLink': u'http://books.google.com/books?id=p_vaPgAACAAJ&dq=isbn:9787544706919&hl=&source=gbs_api',\n", - " u'language': u'zh-CN',\n", - " u'maturityRating': u'NOT_MATURE',\n", - " u'pageCount': 407,\n", - " u'previewLink': u'http://books.google.com/books?id=p_vaPgAACAAJ&dq=isbn:9787544706919&hl=&cd=1&source=gbs_api',\n", - " u'printType': u'BOOK',\n", - " u'publishedDate': u'2009-01-01',\n", - " u'publisher': u'DynoMedia Inc.',\n", - " u'readingModes': {u'image': False, u'text': False},\n", - " u'title': u'\\u5965\\u5df4\\u9a6c\\u56de\\u5fc6\\u5f55'}}],\n", - " u'kind': u'books#volumes',\n", - " u'totalItems': 1}" - ] - } - ], - "prompt_number": 3 - }, - { - "cell_type": "code", - "collapsed": false, - "input": [ - "print r['items'][0]['volumeInfo']['title']" - ], - "language": "python", - "metadata": {}, - "outputs": [ - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\u5965\u5df4\u9a6c\u56de\u5fc6\u5f55\n" - ] - } - ], - "prompt_number": 4 - }, - { - "cell_type": "code", - "collapsed": false, - "input": [ - "item = r['items'][0]\n", - "info = item['accessInfo']\n", - "(item['id'], item.keys() , item)" - ], - "language": "python", - "metadata": {}, - "outputs": [ - { - "metadata": {}, - "output_type": "pyout", - "prompt_number": 5, - "text": [ - "(u'p_vaPgAACAAJ',\n", - " [u'kind',\n", - " u'volumeInfo',\n", - " u'searchInfo',\n", - " u'saleInfo',\n", - " u'etag',\n", - " u'accessInfo',\n", - " u'id',\n", - " u'selfLink'],\n", - " {u'accessInfo': {u'accessViewStatus': u'NONE',\n", - " u'country': u'US',\n", - " u'embeddable': False,\n", - " u'epub': {u'isAvailable': False},\n", - " u'pdf': {u'isAvailable': False},\n", - " u'publicDomain': False,\n", - " u'quoteSharingAllowed': False,\n", - " u'textToSpeechPermission': u'ALLOWED',\n", - " u'viewability': u'NO_PAGES',\n", - " u'webReaderLink': u'http://books.google.com/books/reader?id=p_vaPgAACAAJ&hl=&printsec=frontcover&output=reader&source=gbs_api'},\n", - " u'etag': u'ua9KgTFG/uE',\n", - " u'id': u'p_vaPgAACAAJ',\n", - " u'kind': u'books#volume',\n", - " u'saleInfo': {u'country': u'US',\n", - " u'isEbook': False,\n", - " u'saleability': u'NOT_FOR_SALE'},\n", - " u'searchInfo': {u'textSnippet': u'\\u672c\\u4e66\\u5206\\u4e09\\u90e8\\u5206,\\u7b2c\\u4e00\\u90e8\\u300a\\u6211\\u7684\\u8eab\\u4e16\\u300b\\u8bb2\\u8ff0\\u4e86\\u5965\\u5df4\\u9a6c\\u7684\\u5bb6\\u5ead\\u80cc\\u666f\\u548c\\u4ed6\\u8f9b\\u9178\\u7684\\u7ae5\\u5e74\\u751f\\u6d3b;\\u7b2c\\u4e8c\\u90e8\\u300a\\u829d\\u52a0\\u54e5\\u8d77\\u98de\\u300b\\u63cf\\u8ff0\\u5965\\u5df4\\u9a6c\\u5728\\u829d\\u52a0\\u54e5\\u5357\\u533a\\u4e09\\u5e74\\u7684\\u793e\\u533a\\u5de5\\u4f5c ...'},\n", - " u'selfLink': u'https://www.googleapis.com/books/v1/volumes/p_vaPgAACAAJ',\n", - " u'volumeInfo': {u'allowAnonLogging': False,\n", - " u'authors': [u'\\u5965\\u5df4\\u9a6c'],\n", - " u'canonicalVolumeLink': u'http://books.google.com/books/about/%E5%A5%A5%E5%B7%B4%E9%A9%AC%E5%9B%9E%E5%BF%86%E5%BD%95.html?hl=&id=p_vaPgAACAAJ',\n", - " u'categories': [u'African Americans'],\n", - " u'contentVersion': u'preview-1.0.0',\n", - " u'description': u'\\u672c\\u4e66\\u5206\\u4e09\\u90e8\\u5206,\\u7b2c\\u4e00\\u90e8\\u300a\\u6211\\u7684\\u8eab\\u4e16\\u300b\\u8bb2\\u8ff0\\u4e86\\u5965\\u5df4\\u9a6c\\u7684\\u5bb6\\u5ead\\u80cc\\u666f\\u548c\\u4ed6\\u8f9b\\u9178\\u7684\\u7ae5\\u5e74\\u751f\\u6d3b;\\u7b2c\\u4e8c\\u90e8\\u300a\\u829d\\u52a0\\u54e5\\u8d77\\u98de\\u300b\\u63cf\\u8ff0\\u5965\\u5df4\\u9a6c\\u5728\\u829d\\u52a0\\u54e5\\u5357\\u533a\\u4e09\\u5e74\\u7684\\u793e\\u533a\\u5de5\\u4f5c;\\u7b2c\\u4e09\\u90e8\\u300a\\u68a6\\u56de\\u80af\\u5c3c\\u4e9a\\u300b\\u63cf\\u8ff0\\u5965\\u5df4\\u9a6c\\u4e0e\\u540c\\u7236\\u5f02\\u6bcd\\u7684\\u59b9\\u59b9\\u5728\\u7ebd\\u7ea6\\u89c1\\u9762\\u540e\\u5c55\\u5f00\\u7684\\u80af\\u5c3c\\u4e9a\\u90e8\\u843d\\u5bfb\\u6839\\u4e4b\\u65c5\\u3002',\n", - " u'imageLinks': {u'smallThumbnail': u'http://books.google.com/books/content?id=p_vaPgAACAAJ&printsec=frontcover&img=1&zoom=5&source=gbs_api',\n", - " u'thumbnail': u'http://books.google.com/books/content?id=p_vaPgAACAAJ&printsec=frontcover&img=1&zoom=1&source=gbs_api'},\n", - " u'industryIdentifiers': [{u'identifier': u'7544706915',\n", - " u'type': u'ISBN_10'},\n", - " {u'identifier': u'9787544706919', u'type': u'ISBN_13'}],\n", - " u'infoLink': u'http://books.google.com/books?id=p_vaPgAACAAJ&dq=isbn:9787544706919&hl=&source=gbs_api',\n", - " u'language': u'zh-CN',\n", - " u'maturityRating': u'NOT_MATURE',\n", - " u'pageCount': 407,\n", - " u'previewLink': u'http://books.google.com/books?id=p_vaPgAACAAJ&dq=isbn:9787544706919&hl=&cd=1&source=gbs_api',\n", - " u'printType': u'BOOK',\n", - " u'publishedDate': u'2009-01-01',\n", - " u'publisher': u'DynoMedia Inc.',\n", - " u'readingModes': {u'image': False, u'text': False},\n", - " u'title': u'\\u5965\\u5df4\\u9a6c\\u56de\\u5fc6\\u5f55'}})" - ] - } - ], - "prompt_number": 5 - }, - { - "cell_type": "code", - "collapsed": false, - "input": [ - "def google_info(r):\n", - " if r is not None:\n", - " item = r['items'][0]\n", - " return {\n", - " 'id': item['id'], \n", - " 'title': item['volumeInfo']['title'], \n", - " 'lang': item['volumeInfo']['language'], \n", - " 'identifiers': item['volumeInfo']['industryIdentifiers']\n", - " }\n", - " else:\n", - " return {}\n" - ], - "language": "python", - "metadata": {}, - "outputs": [], - "prompt_number": 6 - }, - { - "cell_type": "code", - "collapsed": false, - "input": [ - "google_info(r)" - ], - "language": "python", - "metadata": {}, - "outputs": [ - { - "metadata": {}, - "output_type": "pyout", - "prompt_number": 7, - "text": [ - "{'id': u'p_vaPgAACAAJ',\n", - " 'identifiers': [{u'identifier': u'7544706915', u'type': u'ISBN_10'},\n", - " {u'identifier': u'9787544706919', u'type': u'ISBN_13'}],\n", - " 'lang': u'zh-CN',\n", - " 'title': u'\\u5965\\u5df4\\u9a6c\\u56de\\u5fc6\\u5f55'}" - ] - } - ], - "prompt_number": 7 - }, - { - "cell_type": "code", - "collapsed": false, - "input": [ - "from itertools import islice\n", - "\n", - "results = []\n", - "\n", - "for isbn_ in islice(bookdata.thingisbn('9789571349268'),None):\n", - " r = bookloader.get_google_isbn_results(isbn_)\n", - " print (isbn_, google_info(r))\n", - " results.append((isbn_, google_info(r)))" - ], - "language": "python", - "metadata": {}, - "outputs": [ - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "('1400082773', {'lang': u'en', 'identifiers': [{u'identifier': u'9781400082773', u'type': u'ISBN_13'}, {u'identifier': u'1400082773', u'type': u'ISBN_10'}], 'id': u'TFwulcQbUDEC', 'title': u'Dreams from My Father'})\n", - "('0307383415', {'lang': u'en', 'identifiers': [{u'identifier': u'9780307383419', u'type': u'ISBN_13'}, {u'identifier': u'0307383415', u'type': u'ISBN_10'}], 'id': u'bmFZl4KAqA0C', 'title': u'Dreams from My Father'})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('1847670946', {'lang': u'en', 'identifiers': [{u'identifier': u'9781847670946', u'type': u'ISBN_13'}, {u'identifier': u'1847670946', u'type': u'ISBN_10'}], 'id': u'Gf7JE4PtNNgC', 'title': u'Dreams from My Father'})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('1921351438', {})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('1847670911', {'lang': u'en', 'identifiers': [{u'identifier': u'9781847670915', u'type': u'ISBN_13'}, {u'identifier': u'1847670911', u'type': u'ISBN_10'}], 'id': u'zUUylKOW8PUC', 'title': u'Dreams from My Father'})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('0739321005', {})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('1847673511', {'lang': u'en', 'identifiers': [{u'identifier': u'9781847673510', u'type': u'ISBN_13'}, {u'identifier': u'1847673511', u'type': u'ISBN_10'}], 'id': u'fflG5gPlUA0C', 'title': u'Dreams from My Father'})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('081292343X', {'lang': u'en', 'identifiers': [{u'identifier': u'081292343X', u'type': u'ISBN_10'}, {u'identifier': u'9780812923438', u'type': u'ISBN_13'}], 'id': u'GSWUPwAACAAJ', 'title': u'Dreams from My Father'})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('9045015102', {})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('2757810898', {'lang': u'fr', 'identifiers': [{u'identifier': u'2757810898', u'type': u'ISBN_10'}, {u'identifier': u'9782757810897', u'type': u'ISBN_13'}], 'id': u'2ilslgEACAAJ', 'title': u'Les r\\xeaves de mon p\\xe8re'})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('0739328190', {'lang': u'en', 'identifiers': [{u'identifier': u'0739328190', u'type': u'ISBN_10'}, {u'identifier': u'9780739328194', u'type': u'ISBN_13'}], 'id': u'Bpw7LwAACAAJ', 'title': u'Dreams from My Father'})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('1847674380', {'lang': u'en', 'identifiers': [{u'identifier': u'9781847674388', u'type': u'ISBN_13'}, {u'identifier': u'1847674380', u'type': u'ISBN_10'}], 'id': u'yya9IL4Y-zcC', 'title': u'Dreams from My Father'})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('9100121967', {})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('0739374710', {})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('904500089X', {})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('0307473872', {'lang': u'es', 'identifiers': [{u'identifier': u'9780307473875', u'type': u'ISBN_13'}, {u'identifier': u'0307473872', u'type': u'ISBN_10'}], 'id': u'7fXUW_2tqdEC', 'title': u'Los Sue\\xf1os de Mi Padre'})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('3423345705', {'lang': u'de', 'identifiers': [{u'identifier': u'3423345705', u'type': u'ISBN_10'}, {u'identifier': u'9783423345705', u'type': u'ISBN_13'}], 'id': u'Ef1tjgEACAAJ', 'title': u'Ein amerikanischer Traum'})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('3446230211', {'lang': u'de', 'identifiers': [{u'identifier': u'3446230211', u'type': u'ISBN_10'}, {u'identifier': u'9783446230217', u'type': u'ISBN_13'}], 'id': u'98jnw152DpUC', 'title': u'Ein amerikanischer Traum'})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('1568361629', {'lang': u'en', 'identifiers': [{u'identifier': u'1568361629', u'type': u'ISBN_10'}, {u'identifier': u'9781568361628', u'type': u'ISBN_13'}], 'id': u'4higAQAACAAJ', 'title': u'Dreams from My Father'})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('9100120650', {})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('1415941394', {})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('0739325760', {'lang': u'en', 'identifiers': [{u'identifier': u'9780739325766', u'type': u'ISBN_13'}, {u'identifier': u'0739325760', u'type': u'ISBN_10'}], 'id': u'hu437EM5jCcC', 'title': u'Dreams from My Father'})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('1417666455', {'lang': u'en', 'identifiers': [{u'identifier': u'1417666455', u'type': u'ISBN_10'}, {u'identifier': u'9781417666454', u'type': u'ISBN_13'}], 'id': u'yRQPNQAACAAJ', 'title': u'Dreams from My Father'})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('2246741416', {})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('2258075971', {'lang': u'fr', 'identifiers': [{u'identifier': u'2258075971', u'type': u'ISBN_10'}, {u'identifier': u'9782258075979', u'type': u'ISBN_13'}], 'id': u'UMg2NQAACAAJ', 'title': u'Les r\\xeaves de mon p\\xe8re'})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('9516927238', {'lang': u'en', 'identifiers': [{u'identifier': u'9516927238', u'type': u'ISBN_10'}, {u'identifier': u'9789516927230', u'type': u'ISBN_13'}], 'id': u'6SgzQwAACAAJ', 'title': u'Unelmia is\\xe4lt\\xe4ni'})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('9100117285', {'lang': u'en', 'identifiers': [{u'identifier': u'9100117285', u'type': u'ISBN_10'}, {u'identifier': u'9789100117283', u'type': u'ISBN_13'}], 'id': u'SSi-NAAACAAJ', 'title': u'Min far hade en dr\\xf6m'})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('8888389865', {'lang': u'it', 'identifiers': [{u'identifier': u'8888389865', u'type': u'ISBN_10'}, {u'identifier': u'9788888389868', u'type': u'ISBN_13'}], 'id': u'JHbSHwAACAAJ', 'title': u'I sogni di mio padre'})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('8703034593', {})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('0320079414', {})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('8493668508', {'lang': u'es', 'identifiers': [{u'identifier': u'8493668508', u'type': u'ISBN_10'}, {u'identifier': u'9788493668501', u'type': u'ISBN_13'}], 'id': u'uflpPgAACAAJ', 'title': u'Los sue\\xf1os de mi padre'})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('1921351012', {'lang': u'en', 'identifiers': [{u'identifier': u'1921351012', u'type': u'ISBN_10'}, {u'identifier': u'9781921351013', u'type': u'ISBN_13'}], 'id': u'ntT1GQAACAAJ', 'title': u'Dreams from My Father'})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('1847673287', {})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('7544706915', {'lang': u'zh-CN', 'identifiers': [{u'identifier': u'7544706915', u'type': u'ISBN_10'}, {u'identifier': u'9787544706919', u'type': u'ISBN_13'}], 'id': u'p_vaPgAACAAJ', 'title': u'\\u5965\\u5df4\\u9a6c\\u56de\\u5fc6\\u5f55'})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('8711318899', {'lang': u'da', 'identifiers': [{u'identifier': u'8711318899', u'type': u'ISBN_10'}, {u'identifier': u'9788711318898', u'type': u'ISBN_13'}], 'id': u'ZMB-twAACAAJ', 'title': u'Arven fra min far'})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('8202297869', {})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('8650510299', {'lang': u'en', 'identifiers': [{u'identifier': u'8650510299', u'type': u'ISBN_10'}, {u'identifier': u'9788650510292', u'type': u'ISBN_13'}], 'id': u'6PUpQwAACAAJ', 'title': u'Snovi moga oca'})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('0739374729', {})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('8184520174', {'lang': u'en', 'identifiers': [{u'identifier': u'8184520174', u'type': u'ISBN_10'}, {u'identifier': u'9788184520170', u'type': u'ISBN_13'}], 'id': u'V1JlPgAACAAJ', 'title': u'Pit\\u0101 se mile sapane'})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('1921520620', {})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('0517194112', {'lang': u'en', 'identifiers': [{u'identifier': u'0517194112', u'type': u'ISBN_10'}, {u'identifier': u'9780517194119', u'type': u'ISBN_13'}], 'id': u'5xigAQAACAAJ', 'title': u'Dreams from My Father'})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('9871117671', {})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('9724618307', {})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('0307394123', {'lang': u'en', 'identifiers': [{u'identifier': u'0307394123', u'type': u'ISBN_10'}, {u'identifier': u'9780307394125', u'type': u'ISBN_13'}], 'id': u'HRCHJp-V0QUC', 'title': u'Dreams from My Father'})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('9571349267', {})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('0812927710', {'lang': u'en', 'identifiers': [{u'identifier': u'0812927710', u'type': u'ISBN_10'}, {u'identifier': u'9780812927719', u'type': u'ISBN_13'}], 'id': u'I2BkkQEACAAJ', 'title': u'Dreams from My Father'})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('1415941408', {})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('1602520275', {'lang': u'en', 'identifiers': [{u'identifier': u'1602520275', u'type': u'ISBN_10'}, {u'identifier': u'9781602520271', u'type': u'ISBN_13'}], 'id': u'l-7BnQEACAAJ', 'title': u'Dreams from My Father'})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('8711413352', {'lang': u'da', 'identifiers': [{u'identifier': u'8711413352', u'type': u'ISBN_10'}, {u'identifier': u'9788711413357', u'type': u'ISBN_13'}], 'id': u'KnKhtwAACAAJ', 'title': u'Arven fra min far'})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('904501386X', {})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('8204142692', {})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('1921569204', {'lang': u'en', 'identifiers': [{u'identifier': u'1921569204', u'type': u'ISBN_10'}, {u'identifier': u'9781921569203', u'type': u'ISBN_13'}], 'id': u'NTU1NAEACAAJ', 'title': u'Dreams from My Father'})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('1602525277', {})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('1921776137', {'lang': u'en', 'identifiers': [{u'identifier': u'9781921776137', u'type': u'ISBN_13'}, {u'identifier': u'1921776137', u'type': u'ISBN_10'}], 'id': u'3fEdaHmtaSsC', 'title': u'Dreams From My Father'})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('1408429152', {'lang': u'en', 'identifiers': [{u'identifier': u'1408429152', u'type': u'ISBN_10'}, {u'identifier': u'9781408429150', u'type': u'ISBN_13'}], 'id': u'Gh-bPwAACAAJ', 'title': u'Dreams from My Father'})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('1471204073', {})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('9100140503', {'lang': u'sv', 'identifiers': [{u'identifier': u'9789100140502', u'type': u'ISBN_13'}, {u'identifier': u'9100140503', u'type': u'ISBN_10'}], 'id': u'sW5dAgAAQBAJ', 'title': u'Min far hade en dr\\xf6m'})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('1921520515', {'lang': u'pt', 'identifiers': [{u'identifier': u'1921520515', u'type': u'ISBN_10'}, {u'identifier': u'9781921520518', u'type': u'ISBN_13'}], 'id': u'R3hIPwAACAAJ', 'title': u'Dreams from My Father'})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('0739321013', {})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('184767352X', {'lang': u'en', 'identifiers': [{u'identifier': u'9781847673527', u'type': u'ISBN_13'}, {u'identifier': u'184767352X', u'type': u'ISBN_10'}], 'id': u'Rpk81dM767kC', 'title': u'Dreams From My Father'})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('1408429160', {})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('3869090227', {})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('1471200930', {})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('3899036093', {})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n", - "('1471215512', {})" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "\n" - ] - } - ], - "prompt_number": 8 - }, - { - "cell_type": "code", - "collapsed": false, - "input": [ - "len(results)" - ], - "language": "python", - "metadata": {}, - "outputs": [ - { - "metadata": {}, - "output_type": "pyout", - "prompt_number": 9, - "text": [ - "65" - ] - } - ], - "prompt_number": 9 - }, - { - "cell_type": "code", - "collapsed": false, - "input": [ - "for (isbn_, result) in results:\n", - " print (isbn_, result.get('lang'))" - ], - "language": "python", - "metadata": {}, - "outputs": [ - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "('1400082773', u'en')\n", - "('0307383415', u'en')\n", - "('1847670946', u'en')\n", - "('1921351438', None)\n", - "('1847670911', u'en')\n", - "('0739321005', None)\n", - "('1847673511', u'en')\n", - "('081292343X', u'en')\n", - "('9045015102', None)\n", - "('2757810898', u'fr')\n", - "('0739328190', u'en')\n", - "('1847674380', u'en')\n", - "('9100121967', None)\n", - "('0739374710', None)\n", - "('904500089X', None)\n", - "('0307473872', u'es')\n", - "('3423345705', u'de')\n", - "('3446230211', u'de')\n", - "('1568361629', u'en')\n", - "('9100120650', None)\n", - "('1415941394', None)\n", - "('0739325760', u'en')\n", - "('1417666455', u'en')\n", - "('2246741416', None)\n", - "('2258075971', u'fr')\n", - "('9516927238', u'en')\n", - "('9100117285', u'en')\n", - "('8888389865', u'it')\n", - "('8703034593', None)\n", - "('0320079414', None)\n", - "('8493668508', u'es')\n", - "('1921351012', u'en')\n", - "('1847673287', None)\n", - "('7544706915', u'zh-CN')\n", - "('8711318899', u'da')\n", - "('8202297869', None)\n", - "('8650510299', u'en')\n", - "('0739374729', None)\n", - "('8184520174', u'en')\n", - "('1921520620', None)\n", - "('0517194112', u'en')\n", - "('9871117671', None)\n", - "('9724618307', None)\n", - "('0307394123', u'en')\n", - "('9571349267', None)\n", - "('0812927710', u'en')\n", - "('1415941408', None)\n", - "('1602520275', u'en')\n", - "('8711413352', u'da')\n", - "('904501386X', None)\n", - "('8204142692', None)\n", - "('1921569204', u'en')\n", - "('1602525277', None)\n", - "('1921776137', u'en')\n", - "('1408429152', u'en')\n", - "('1471204073', None)\n", - "('9100140503', u'sv')\n", - "('1921520515', u'pt')\n", - "('0739321013', None)\n", - "('184767352X', u'en')\n", - "('1408429160', None)\n", - "('3869090227', None)\n", - "('1471200930', None)\n", - "('3899036093', None)\n", - "('1471215512', None)\n" - ] - } - ], - "prompt_number": 10 - }, - { - "cell_type": "code", - "collapsed": false, - "input": [ - "k = isbn.ISBN('7544706915')\n", - "k.to_string()" - ], - "language": "python", - "metadata": {}, - "outputs": [ - { - "metadata": {}, - "output_type": "pyout", - "prompt_number": 11, - "text": [ - "'9787544706919'" - ] - } - ], - "prompt_number": 11 - } - ], - "metadata": {} + "cell_type": "markdown", + "metadata": {}, + "source": [ + "http://jenkins.unglueit.com/job/regluit/3550/console\n", + "\n", + "looking for book to replace the problematic one" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from regluit.experimental import bookdata\n", + "from regluit.core import isbn" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from regluit.core import bookloader\n", + "\n", + "#edition = bookloader.add_by_isbn('9789571349268')\n", + "#assert edition.work.language == 'zh-TW'\n", + "\n", + "edition = bookloader.add_by_isbn('9787544706919')\n", + "assert edition.work.language == 'zh-CN'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "r = bookloader.get_google_isbn_results('9787544706919')\n", + "r" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "print r['items'][0]['volumeInfo']['title']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "item = r['items'][0]\n", + "info = item['accessInfo']\n", + "(item['id'], item.keys() , item)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "def google_info(r):\n", + " if r is not None:\n", + " item = r['items'][0]\n", + " return {\n", + " 'id': item['id'], \n", + " 'title': item['volumeInfo']['title'], \n", + " 'lang': item['volumeInfo']['language'], \n", + " 'identifiers': item['volumeInfo']['industryIdentifiers']\n", + " }\n", + " else:\n", + " return {}\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "google_info(r)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from itertools import islice\n", + "\n", + "results = []\n", + "\n", + "for isbn_ in islice(bookdata.thingisbn('9789571349268'),None):\n", + " r = bookloader.get_google_isbn_results(isbn_)\n", + " print (isbn_, google_info(r))\n", + " results.append((isbn_, google_info(r)))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "len(results)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "for (isbn_, result) in results:\n", + " print (isbn_, result.get('lang'))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "k = isbn.ISBN('7544706915')\n", + "k.to_string()" + ] } - ] -} \ No newline at end of file + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.11" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile index 0d3749ae..3dd6ba6d 100644 --- a/vagrant/Vagrantfile +++ b/vagrant/Vagrantfile @@ -30,7 +30,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| "-e class=please", "-e hostname=please.unglue.it", "-e setdns=true", - "-e branch=nosudo_celery" + "-e branch=master" ] end