implementing tests for facets

theme-version-to-0.4.x
Safwan Rahman 2018-05-28 01:40:59 +06:00
parent c6f39e7680
commit 085d5baf42
10 changed files with 212 additions and 37 deletions

View File

@ -1,10 +1,10 @@
import json
import pytest
from django_dynamic_fixture import G
from faker import Faker
from readthedocs.projects.models import Project
from readthedocs.search.indexes import Index, ProjectIndex, PageIndex, SectionIndex
from .dummy_data import DUMMY_PAGE_JSON, ALL_PROJECTS
fake = Faker()
@ -35,44 +35,26 @@ def search():
@pytest.fixture
def project():
return G(Project)
def all_projects():
return [G(Project, slug=project_name, name=project_name) for project_name in ALL_PROJECTS]
@pytest.fixture
def page_json():
version_contents = {}
def project(all_projects):
# Return a single project
return all_projects[0]
def create_dummy_json():
data = {
'path': fake.word(),
'title': fake.sentence(),
'content': fake.text(),
'sections': fake.sentences(),
'headers': fake.sentences()
}
return data
def get_dummy_json(version, *args, **kwargs):
"""Get dummy json content for a version page"""
# Check existing content of that version
# If not exist, generate new dummy content
content = version_contents.get(version.id)
if not content:
content = create_dummy_json()
# save in order to not regenerate dummy content for same version
version_contents[version.id] = content
return [content]
return get_dummy_json
def get_dummy_page_json(version, *args, **kwargs):
dummy_page_json = DUMMY_PAGE_JSON
project_name = version.project.name
return dummy_page_json.get(project_name)
@pytest.fixture
def mock_parse_json(mocker, page_json):
def mock_parse_json(mocker):
# patch the function from `projects.tasks` because it has been point to there
# http://www.voidspace.org.uk/python/mock/patch.html#where-to-patch
mocked_function = mocker.patch('readthedocs.projects.tasks.process_all_json_files')
mocked_function.side_effect = page_json
mocked_function.side_effect = get_dummy_page_json

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1,22 @@
{
"content": "Documentation\nThis documentation is generated and published at Read the Docs whenever the master branch is updated.\nGitHub can render our .rst documents as ReStructuredText, which is close enough to Sphinx for most code reviews, without features like links between documents.\nIt is occasionally necessary to generate the documentation locally. It is easiest to do this with a virtualenv on the host system, using Docker only to regenerate the MDN Sphinx template. If you are not comfortable with that style of development, it can be done entirely in Docker using docker-compose.\nGenerating documentation\nSphinx uses a Makefile in the docs subfolder to build documentation in several formats. MDN only uses the HTML format, and the generated document index is at docs/_build/html/index.html.\nTo generate the documentation in a virtualenv on the host machine, first install the requirements:\npip install -r requirements/docs.txt\nThen switch to the docs folder to use the Makefile:\ncd docs make html python -m webbrowser file://${PWD}/_build/html/index.html\nTo generate the documentation with Docker:\ndocker-compose run --rm --user $(id -u) web sh -c \"\\ virtualenv /tmp/.venvs/docs && \\ . /tmp/.venvs/docs/bin/activate && \\ pip install -r /app/requirements/docs.txt && \\ cd /app/docs && \\ make html\" python -m webbrowser file://${PWD}/docs/_build/html/index.html\nA virtualenv is required, to avoid a pip bug when changing the version of a system-installed package.",
"headers": [
"Documentation",
"Generating documentation",
"Installation"
],
"title": "Documentation",
"sections": [
{
"content": "\nThis documentation is generated and published at\n<a class=\"reference external\" href=\"https://kuma.readthedocs.io/en/latest/\">Read the Docs</a> whenever the master branch is updated.\n\nGitHub can render our <code class=\"docutils literal notranslate\"><span class=\"pre\">.rst</span></code> documents as <a class=\"reference external\" href=\"https://en.wikipedia.org/wiki/ReStructuredText\">ReStructuredText</a>, which is\nclose enough to <a class=\"reference external\" href=\"https://en.wikipedia.org/wiki/Sphinx_(documentation_generator)\">Sphinx</a> for most code reviews, without features like links\nbetween documents.\n\nIt is occasionally necessary to generate the documentation locally. It is\neasiest to do this with a <a class=\"reference external\" href=\"https://virtualenv.pypa.io/en/stable/\">virtualenv</a> on the host system, using Docker only to\nregenerate the MDN Sphinx template. If you are not comfortable with that style\nof development, it can be done entirely in Docker using <code class=\"docutils literal notranslate\"><span class=\"pre\">docker-compose</span></code>.\n",
"id": "documentation",
"title": "Documentation"
},
{
"content": "\n<h2>Generating documentation<a class=\"headerlink\" href=\"#generating-documentation\" title=\"Permalink to this headline\">\u00b6</a></h2>\n<p>Sphinx uses a <code class=\"docutils literal notranslate\"><span class=\"pre\">Makefile</span></code> in the <code class=\"docutils literal notranslate\"><span class=\"pre\">docs</span></code> subfolder to build documentation in\nseveral formats. MDN only uses the HTML format, and the generated document\nindex is at <code class=\"docutils literal notranslate\"><span class=\"pre\">docs/_build/html/index.html</span></code>.</p>\n<p>To generate the documentation in a virtualenv on the host machine, first\ninstall the requirements:</p>\n<div class=\"highlight-default notranslate\"><div class=\"highlight\"><pre><span/><span class=\"n\">pip</span> <span class=\"n\">install</span> <span class=\"o\">-</span><span class=\"n\">r</span> <span class=\"n\">requirements</span><span class=\"o\">/</span><span class=\"n\">docs</span><span class=\"o\">.</span><span class=\"n\">txt</span>\n</pre></div>\n</div>\n<p>Then switch to the <code class=\"docutils literal notranslate\"><span class=\"pre\">docs</span></code> folder to use the <code class=\"docutils literal notranslate\"><span class=\"pre\">Makefile</span></code>:</p>\n<div class=\"highlight-default notranslate\"><div class=\"highlight\"><pre><span/>cd docs\nmake html\npython -m webbrowser file://${PWD}/_build/html/index.html\n</pre></div>\n</div>\n<p>To generate the documentation with Docker:</p>\n<div class=\"highlight-default notranslate\"><div class=\"highlight\"><pre><span/>docker-compose run --rm --user $(id -u) web sh -c \"\\\n virtualenv /tmp/.venvs/docs &amp;&amp; \\\n . /tmp/.venvs/docs/bin/activate &amp;&amp; \\\n pip install -r /app/requirements/docs.txt &amp;&amp; \\\n cd /app/docs &amp;&amp; \\\n make html\"\npython -m webbrowser file://${PWD}/docs/_build/html/index.html\n</pre></div>\n</div>\n<p>A <code class=\"docutils literal notranslate\"><span class=\"pre\">virtualenv</span></code> is required, to avoid a <code class=\"docutils literal notranslate\"><span class=\"pre\">pip</span></code> bug when changing the version\nof a system-installed package.</p>\n",
"id": "generating-documentation",
"title": "Generating documentation"
}
],
"path": "documentation"
}

View File

@ -0,0 +1,33 @@
{
"content": "Installation\nEither check out Pipeline from GitHub or to pull a release off PyPI\npip install django-pipeline\nAdd \u2018pipeline\u2019 to your INSTALLED_APPS\nINSTALLED_APPS = ( 'pipeline', )\nUse a pipeline storage for STATICFILES_STORAGE\nSTATICFILES_STORAGE = 'pipeline.storage.PipelineCachedStorage'\nAdd the PipelineFinder to STATICFILES_FINDERS\nSTATICFILES_FINDERS = ( 'django.contrib.staticfiles.finders.FileSystemFinder', 'django.contrib.staticfiles.finders.AppDirectoriesFinder', 'pipeline.finders.PipelineFinder', )\nNote\nYou need to use Django>=1.7 to be able to use this version of pipeline.\nUpgrading from 1.3\nTo upgrade from pipeline 1.3, you will need to follow these steps:\nUpdate templates to use the new syntax\n{# pipeline<1.4 #} {% load compressed %} {% compressed_js 'group' %} {% compressed_css 'group' %}\n{# pipeline>=1.4 #} {% load pipeline %} {% javascript 'group' %} {% stylesheet 'group' %}\nAdd the PipelineFinder to STATICFILES_FINDERS\nSTATICFILES_FINDERS = ( 'django.contrib.staticfiles.finders.FileSystemFinder', 'django.contrib.staticfiles.finders.AppDirectoriesFinder', 'pipeline.finders.PipelineFinder', )\nUpgrading from 1.5\nTo upgrade from pipeline 1.5, you will need update all your PIPELINE_* settings and move them under the new PIPELINE setting. See Configuration.\nRecommendations\nPipeline\u2019s default CSS and JS compressor is Yuglify. Yuglify wraps UglifyJS and cssmin, applying the default YUI configurations to them. It can be downloaded from: https://github.com/yui/yuglify/.\nIf you do not install yuglify, make sure to disable the compressor in your settings.",
"headers": [
"Installation",
"Upgrading from 1.3",
"Upgrading from 1.5",
"Recommendations"
],
"title": "Installation",
"sections": [
{
"content": "\n\n<li><p class=\"first\">Either check out Pipeline from <a class=\"reference external\" href=\"http://github.com/jazzband/django-pipeline\">GitHub</a> or to pull a release off <a class=\"reference external\" href=\"http://pypi.python.org/pypi/django-pipeline\">PyPI</a></p>\n<div class=\"highlight-default notranslate\"><div class=\"highlight\"><pre><span/><span class=\"n\">pip</span> <span class=\"n\">install</span> <span class=\"n\">django</span><span class=\"o\">-</span><span class=\"n\">pipeline</span>\n</pre></div>\n</div>\n</li>\n<li><p class=\"first\">Add \u2018pipeline\u2019 to your <code class=\"docutils literal notranslate\"><span class=\"pre\">INSTALLED_APPS</span></code></p>\n<div class=\"highlight-default notranslate\"><div class=\"highlight\"><pre><span/><span class=\"n\">INSTALLED_APPS</span> <span class=\"o\">=</span> <span class=\"p\">(</span>\n <span class=\"s1\">'pipeline'</span><span class=\"p\">,</span>\n<span class=\"p\">)</span>\n</pre></div>\n</div>\n</li>\n<li><p class=\"first\">Use a pipeline storage for <code class=\"docutils literal notranslate\"><span class=\"pre\">STATICFILES_STORAGE</span></code></p>\n<div class=\"highlight-default notranslate\"><div class=\"highlight\"><pre><span/><span class=\"n\">STATICFILES_STORAGE</span> <span class=\"o\">=</span> <span class=\"s1\">'pipeline.storage.PipelineCachedStorage'</span>\n</pre></div>\n</div>\n</li>\n<li><p class=\"first\">Add the <code class=\"docutils literal notranslate\"><span class=\"pre\">PipelineFinder</span></code> to <code class=\"docutils literal notranslate\"><span class=\"pre\">STATICFILES_FINDERS</span></code></p>\n<div class=\"highlight-default notranslate\"><div class=\"highlight\"><pre><span/><span class=\"n\">STATICFILES_FINDERS</span> <span class=\"o\">=</span> <span class=\"p\">(</span>\n <span class=\"s1\">'django.contrib.staticfiles.finders.FileSystemFinder'</span><span class=\"p\">,</span>\n <span class=\"s1\">'django.contrib.staticfiles.finders.AppDirectoriesFinder'</span><span class=\"p\">,</span>\n <span class=\"s1\">'pipeline.finders.PipelineFinder'</span><span class=\"p\">,</span>\n<span class=\"p\">)</span>\n</pre></div>\n</div>\n</li>\n\n\n\n<p class=\"first admonition-title\">Note</p>\n<p class=\"last\">You need to use <code class=\"docutils literal notranslate\"><span class=\"pre\">Django&gt;=1.7</span></code> to be able to use this version of pipeline.</p>\n\n",
"id": "installation",
"title": "Installation"
},
{
"content": "\n<h2>Upgrading from 1.3<a class=\"headerlink\" href=\"#upgrading-from-1-3\" title=\"Permalink to this headline\">\u00b6</a></h2>\n<p>To upgrade from pipeline 1.3, you will need to follow these steps:</p>\n<ol class=\"arabic\">\n<li><p class=\"first\">Update templates to use the new syntax</p>\n<blockquote>\n<div><div class=\"highlight-python notranslate\"><div class=\"highlight\"><pre><span/><span class=\"p\">{</span><span class=\"c1\"># pipeline&lt;1.4 #}</span>\n<span class=\"p\">{</span><span class=\"o\">%</span> <span class=\"n\">load</span> <span class=\"n\">compressed</span> <span class=\"o\">%</span><span class=\"p\">}</span>\n<span class=\"p\">{</span><span class=\"o\">%</span> <span class=\"n\">compressed_js</span> <span class=\"s1\">'group'</span> <span class=\"o\">%</span><span class=\"p\">}</span>\n<span class=\"p\">{</span><span class=\"o\">%</span> <span class=\"n\">compressed_css</span> <span class=\"s1\">'group'</span> <span class=\"o\">%</span><span class=\"p\">}</span>\n</pre></div>\n</div>\n<div class=\"highlight-python notranslate\"><div class=\"highlight\"><pre><span/><span class=\"p\">{</span><span class=\"c1\"># pipeline&gt;=1.4 #}</span>\n<span class=\"p\">{</span><span class=\"o\">%</span> <span class=\"n\">load</span> <span class=\"n\">pipeline</span> <span class=\"o\">%</span><span class=\"p\">}</span>\n<span class=\"p\">{</span><span class=\"o\">%</span> <span class=\"n\">javascript</span> <span class=\"s1\">'group'</span> <span class=\"o\">%</span><span class=\"p\">}</span>\n<span class=\"p\">{</span><span class=\"o\">%</span> <span class=\"n\">stylesheet</span> <span class=\"s1\">'group'</span> <span class=\"o\">%</span><span class=\"p\">}</span>\n</pre></div>\n</div>\n</div></blockquote>\n</li>\n<li><p class=\"first\">Add the <code class=\"docutils literal notranslate\"><span class=\"pre\">PipelineFinder</span></code> to <code class=\"docutils literal notranslate\"><span class=\"pre\">STATICFILES_FINDERS</span></code></p>\n<div class=\"highlight-default notranslate\"><div class=\"highlight\"><pre><span/><span class=\"n\">STATICFILES_FINDERS</span> <span class=\"o\">=</span> <span class=\"p\">(</span>\n <span class=\"s1\">'django.contrib.staticfiles.finders.FileSystemFinder'</span><span class=\"p\">,</span>\n <span class=\"s1\">'django.contrib.staticfiles.finders.AppDirectoriesFinder'</span><span class=\"p\">,</span>\n <span class=\"s1\">'pipeline.finders.PipelineFinder'</span><span class=\"p\">,</span>\n<span class=\"p\">)</span>\n</pre></div>\n</div>\n</li>\n</ol>\n",
"id": "upgrading-from-1-3",
"title": "Upgrading from 1.3"
},
{
"content": "\n<h2>Upgrading from 1.5<a class=\"headerlink\" href=\"#upgrading-from-1-5\" title=\"Permalink to this headline\">\u00b6</a></h2>\n<p>To upgrade from pipeline 1.5, you will need update all your <code class=\"docutils literal notranslate\"><span class=\"pre\">PIPELINE_*</span></code>\nsettings and move them under the new <code class=\"docutils literal notranslate\"><span class=\"pre\">PIPELINE</span></code> setting.\nSee <a class=\"reference internal\" href=\"../configuration/#ref-configuration\"><span class=\"std std-ref\">Configuration</span></a>.</p>\n",
"id": "upgrading-from-1-5",
"title": "Upgrading from 1.5"
},
{
"content": "\n<h2>Recommendations<a class=\"headerlink\" href=\"#recommendations\" title=\"Permalink to this headline\">\u00b6</a></h2>\n<p>Pipeline\u2019s default CSS and JS compressor is Yuglify.\nYuglify wraps UglifyJS and cssmin, applying the default YUI configurations to them.\nIt can be downloaded from: <a class=\"reference external\" href=\"https://github.com/yui/yuglify/\">https://github.com/yui/yuglify/</a>.</p>\n<p>If you do not install yuglify, make sure to disable the compressor in your settings.</p>\n",
"id": "recommendations",
"title": "Recommendations"
}
],
"path": "installation"
}

View File

@ -0,0 +1,27 @@
{
"content": "Signals\nList of all signals sent by pipeline.\ncss_compressed\npipeline.signals.css_compressed\nWhenever a css package is compressed, this signal is sent after the compression.\nArguments sent with this signal :\nsender:\nThe Packager class that compressed the group.\npackage:\nThe package actually compressed.\njs_compressed\npipeline.signals.js_compressed\nWhenever a js package is compressed, this signal is sent after the compression.\nArguments sent with this signal :\nsender:\nThe Packager class that compressed the group.\npackage:\nThe package actually compressed.",
"headers": [
"Signals",
"css_compressed",
"js_compressed"
],
"title": "Signals",
"sections": [
{
"content": "\nList of all signals sent by pipeline.\n",
"id": "signals",
"title": "Signals"
},
{
"content": "\n<h2>css_compressed<a class=\"headerlink\" href=\"#css-compressed\" title=\"Permalink to this headline\">\u00b6</a></h2>\n<p><strong>pipeline.signals.css_compressed</strong></p>\n<blockquote>\n<div><p>Whenever a css package is compressed, this signal is sent after the compression.</p>\n<p>Arguments sent with this signal :</p>\n<blockquote>\n<div><table class=\"docutils field-list\" frame=\"void\" rules=\"none\">\n<col class=\"field-name\"/>\n<col class=\"field-body\"/>\n<tbody valign=\"top\">\n<tr class=\"field-odd field\"><th class=\"field-name\">sender:</th><td class=\"field-body\">The <code class=\"docutils literal notranslate\"><span class=\"pre\">Packager</span></code> class that compressed the group.</td>\n</tr>\n<tr class=\"field-even field\"><th class=\"field-name\">package:</th><td class=\"field-body\">The package actually compressed.</td>\n</tr>\n</tbody>\n</table>\n</div></blockquote>\n</div></blockquote>\n",
"id": "css-compressed",
"title": "css_compressed"
},
{
"content": "\n<h2>js_compressed<a class=\"headerlink\" href=\"#js-compressed\" title=\"Permalink to this headline\">\u00b6</a></h2>\n<p><strong>pipeline.signals.js_compressed</strong></p>\n<blockquote>\n<div><p>Whenever a js package is compressed, this signal is sent after the compression.</p>\n<p>Arguments sent with this signal :</p>\n<blockquote>\n<div><table class=\"docutils field-list\" frame=\"void\" rules=\"none\">\n<col class=\"field-name\"/>\n<col class=\"field-body\"/>\n<tbody valign=\"top\">\n<tr class=\"field-odd field\"><th class=\"field-name\">sender:</th><td class=\"field-body\">The <code class=\"docutils literal notranslate\"><span class=\"pre\">Packager</span></code> class that compressed the group.</td>\n</tr>\n<tr class=\"field-even field\"><th class=\"field-name\">package:</th><td class=\"field-body\">The package actually compressed.</td>\n</tr>\n</tbody>\n</table>\n</div></blockquote>\n</div></blockquote>\n",
"id": "js-compressed",
"title": "js_compressed"
}
],
"path": "signals"
}

View File

@ -0,0 +1,27 @@
import json
import os
_DATA_FILES = {
'pipeline': ['installation.json', 'signals.json'],
'kuma': ['documentation.json', 'docker.json'],
}
def _get_dummy_json():
dictionary = {}
for key, value in _DATA_FILES.items():
data = []
for file_name in value:
current_path = os.path.abspath(os.path.dirname(__file__))
path = os.path.join(current_path, "data", key, file_name)
with open(path) as f:
content = json.load(f)
data.append(content)
dictionary[key] = data
return dictionary
DUMMY_PAGE_JSON = _get_dummy_json()
ALL_PROJECTS = DUMMY_PAGE_JSON.keys()

View File

@ -4,9 +4,11 @@ from django.core.urlresolvers import reverse
from django_dynamic_fixture import G
from pyquery import PyQuery as pq
from readthedocs.builds.constants import LATEST
from readthedocs.builds.models import Version
from readthedocs.projects.models import Project
from readthedocs.search import parse_json
from .dummy_data import DUMMY_PAGE_JSON
@pytest.mark.django_db
@ -15,7 +17,7 @@ class TestElasticSearch(object):
url = reverse('search')
@pytest.fixture(autouse=True)
def elastic_index(self, mock_parse_json, project, search):
def elastic_index(self, mock_parse_json, all_projects, search):
call_command('reindex_elasticsearch')
search.refresh_index()
@ -27,13 +29,13 @@ class TestElasticSearch(object):
content = page.find('.module-list-wrapper .module-item-title')
assert project.name.encode('utf-8') in content.text().encode('utf-8')
def test_search_by_file_content(self, client, page_json, project):
def test_search_by_file_content(self, client, project):
versions = project.versions.all()
# There should be only one version of the project
assert len(versions) == 1
data = page_json(version=versions[0])[0]
data = DUMMY_PAGE_JSON[project.slug][0]
# Query with the first word of title
title = data['title']
query = title.split()[0]
@ -44,3 +46,59 @@ class TestElasticSearch(object):
page = pq(resp.content)
content = page.find('.module-list-wrapper .module-item-title')
assert title in content.text()
def test_file_search_show_projects(self, client):
"""Test that search result page shows list of projects while searching for files"""
# `Installation` word is present both in `kuma` and `pipeline` files
# so search with this phrase
resp = client.get(self.url, {'q': "Installation", 'type': 'file'})
assert resp.status_code == 200
page = pq(resp.content)
content = page.find('.module-list-wrapper .module-item-title')
# There should be 2 search result
assert len(content) == 2
# there should be 2 projects in the left side column
content = page.find('.navigable .project-list')
assert len(content) == 2
text = content.text()
# kuma and pipeline should be there
assert 'kuma' and 'pipeline' in text
@pytest.mark.xfail(reason="Versions are not showing correctly! Fixme while rewrite!")
def test_file_search_show_versions(self, client, all_projects, search, settings):
# override the settings to index all versions
settings.INDEX_ONLY_LATEST = False
project = all_projects[0]
# Create some versions of the project
versions = [G(Version, project=project) for _ in range(3)]
call_command('reindex_elasticsearch')
search.refresh_index()
data = DUMMY_PAGE_JSON[project.slug][0]
title = data['title']
query = title.split()[0]
resp = client.get(self.url, {'q': query, 'type': 'file'})
assert resp.status_code == 200
page = pq(resp.content)
content = page.find('.navigable .version-list')
# There should be total 4 versions
# one is latest, and other 3 that we created above
assert len(content) == 4
project_versions = [v.slug for v in versions] + [LATEST]
content_versions = []
for element in content:
text = element.text_content()
# strip and split to keep the version slug only
slug = text.strip().split('\n')[0]
content_versions.append(slug)
assert sorted(project_versions) == sorted(content_versions)

View File

@ -66,6 +66,7 @@ def elastic_search(request):
for term in results['facets'][facet_type]['terms']:
facets[facet_type][term['term']] = term['count']
if settings.DEBUG:
print(pprint(results))
print(pprint(facets))

View File

@ -29,8 +29,8 @@ class CommunityDevSettings(CommunityBaseSettings):
SLUMBER_USERNAME = 'test'
SLUMBER_PASSWORD = 'test' # noqa: ignore dodgy check
SLUMBER_API_HOST = 'http://localhost:8000'
PUBLIC_API_URL = 'http://localhost:8000'
SLUMBER_API_HOST = 'http://127.0.0.1:8000'
PUBLIC_API_URL = 'http://127.0.0.1:8000'
BROKER_URL = 'redis://localhost:6379/0'
CELERY_RESULT_BACKEND = 'redis://localhost:6379/0'

View File

@ -46,7 +46,7 @@
{% if facets.project %}
<h5> Projects </h5>
{% for name, count in facets.project.items %}
<li class="{% if project == name %}active{% endif %}">
<li class="{% if project == name %}active{% endif %} project-list">
{% if project == name %}
<a href="?{% url_replace request 'project' '' %}">{{ name }}
{% else %}
@ -63,7 +63,7 @@
<h5>Version</h5>
{% for name, count in facets.version.items %}
<li class="{% if version == name %}active{% endif %}">
<li class="{% if version == name %}active{% endif %} version-list">
{% if version == name %}
<a href="?{% url_replace request 'version' '' %}">{{ name }}
{% else %}