From eb4faa43ee89b80e52d1566804e050e5a601464f Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 22 Aug 2018 11:13:41 -0300 Subject: [PATCH] Build JSON artifacts in HTML builder This removes the Feature BUILD_JSON_ARTIFACTS_WITH_HTML and makes this the default for all the projects. It also removes the SearchBuilder since it's not used anymore. --- readthedocs/doc_builder/backends/sphinx.py | 47 +++++++------------ readthedocs/doc_builder/loader.py | 1 - .../templates/doc_builder/conf.py.tmpl | 2 +- readthedocs/projects/models.py | 3 -- readthedocs/projects/tasks.py | 19 -------- .../rtd_tests/tests/test_doc_builder.py | 41 +--------------- 6 files changed, 20 insertions(+), 93 deletions(-) diff --git a/readthedocs/doc_builder/backends/sphinx.py b/readthedocs/doc_builder/backends/sphinx.py index d73010ec4..00656f265 100644 --- a/readthedocs/doc_builder/backends/sphinx.py +++ b/readthedocs/doc_builder/backends/sphinx.py @@ -136,9 +136,6 @@ class BaseSphinx(BaseBuilder): 'display_gitlab': display_gitlab, # Features - 'generate_json_artifacts': self.project.has_feature( - Feature.BUILD_JSON_ARTIFACTS_WITH_HTML - ), 'dont_overwrite_sphinx_context': self.project.has_feature( Feature.DONT_OVERWRITE_SPHINX_CONTEXT ), @@ -228,27 +225,26 @@ class HtmlBuilder(BaseSphinx): def move(self, **__): super(HtmlBuilder, self).move() - if self.project.has_feature(Feature.BUILD_JSON_ARTIFACTS_WITH_HTML): - # Copy json artifacts to its own directory - # to keep compatibility with the older builder. - json_path = os.path.abspath( - os.path.join(self.old_artifact_path, '..', 'json') + # Copy JSON artifacts to its own directory + # to keep compatibility with the older builder. + json_path = os.path.abspath( + os.path.join(self.old_artifact_path, '..', 'json') + ) + json_path_target = self.project.artifact_path( + version=self.version.slug, type_='sphinx_search' + ) + if os.path.exists(json_path): + if os.path.exists(json_path_target): + shutil.rmtree(json_path_target) + log.info('Copying json on the local filesystem') + shutil.copytree( + json_path, + json_path_target ) - json_path_target = self.project.artifact_path( - version=self.version.slug, type_='sphinx_search' + else: + log.warning( + 'Not moving json because the build dir is unknown.' ) - if os.path.exists(json_path): - if os.path.exists(json_path_target): - shutil.rmtree(json_path_target) - log.info('Copying json on the local filesystem') - shutil.copytree( - json_path, - json_path_target - ) - else: - log.warning( - 'Not moving json because the build dir is unknown.' - ) class HtmlDirBuilder(HtmlBuilder): @@ -267,13 +263,6 @@ class SingleHtmlBuilder(HtmlBuilder): self.sphinx_builder = 'readthedocssinglehtml' -class SearchBuilder(BaseSphinx): - type = 'sphinx_search' - sphinx_builder = 'json' - sphinx_build_dir = '_build/json' - ignore_patterns = ['_static'] - - class LocalMediaBuilder(BaseSphinx): type = 'sphinx_localmedia' sphinx_builder = 'readthedocssinglehtmllocalmedia' diff --git a/readthedocs/doc_builder/loader.py b/readthedocs/doc_builder/loader.py index 067f475c6..0edcaace7 100644 --- a/readthedocs/doc_builder/loader.py +++ b/readthedocs/doc_builder/loader.py @@ -20,7 +20,6 @@ BUILDER_BY_NAME = { # Other Sphinx Builders 'sphinx_pdf': sphinx.PdfBuilder, 'sphinx_epub': sphinx.EpubBuilder, - 'sphinx_search': sphinx.SearchBuilder, 'sphinx_singlehtmllocalmedia': sphinx.LocalMediaBuilder, # Other markup 'mkdocs': mkdocs.MkdocsHTML, diff --git a/readthedocs/doc_builder/templates/doc_builder/conf.py.tmpl b/readthedocs/doc_builder/templates/doc_builder/conf.py.tmpl index e5e2b4d44..a2b8ed7b2 100644 --- a/readthedocs/doc_builder/templates/doc_builder/conf.py.tmpl +++ b/readthedocs/doc_builder/templates/doc_builder/conf.py.tmpl @@ -143,4 +143,4 @@ else: extensions = ["readthedocs_ext.readthedocs"] # Build the json artifacts with the html build step -rtd_generate_json_artifacts = {{ generate_json_artifacts }} +rtd_generate_json_artifacts = True diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index 83f198fb5..f189f7898 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -1040,7 +1040,6 @@ class Feature(models.Model): ALLOW_DEPRECATED_WEBHOOKS = 'allow_deprecated_webhooks' PIP_ALWAYS_UPGRADE = 'pip_always_upgrade' SKIP_SUBMODULES = 'skip_submodules' - BUILD_JSON_ARTIFACTS_WITH_HTML = 'build_json_artifacts_with_html' DONT_OVERWRITE_SPHINX_CONTEXT = 'dont_overwrite_sphinx_context' ALLOW_V2_CONFIG_FILE = 'allow_v2_config_file' @@ -1050,8 +1049,6 @@ class Feature(models.Model): (ALLOW_DEPRECATED_WEBHOOKS, _('Allow deprecated webhook views')), (PIP_ALWAYS_UPGRADE, _('Always run pip install --upgrade')), (SKIP_SUBMODULES, _('Skip git submodule checkout')), - (BUILD_JSON_ARTIFACTS_WITH_HTML, _( - 'Build the json artifacts with the html build step')), (DONT_OVERWRITE_SPHINX_CONTEXT, _( 'Do not overwrite context vars in conf.py with Read the Docs context',)), (ALLOW_V2_CONFIG_FILE, _( diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index c41154c5f..ce7f8d41f 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -34,7 +34,6 @@ from readthedocs.builds.constants import ( from readthedocs.builds.models import APIVersion, Build, Version from readthedocs.builds.signals import build_complete from readthedocs.builds.syncers import Syncer -from readthedocs.config import ConfigError from readthedocs.core.resolver import resolve_path from readthedocs.core.symlink import PublicSymlink, PrivateSymlink from readthedocs.core.utils import send_email, broadcast @@ -682,7 +681,6 @@ class UpdateDocsTaskStep(SyncRepositoryMixin): version=self.version, max_lock_age=getattr(settings, 'REPO_LOCK_SECONDS', 30)): outcomes['html'] = self.build_docs_html() - outcomes['search'] = self.build_docs_search() outcomes['localmedia'] = self.build_docs_localmedia() outcomes['pdf'] = self.build_docs_pdf() outcomes['epub'] = self.build_docs_epub() @@ -714,23 +712,6 @@ class UpdateDocsTaskStep(SyncRepositoryMixin): return success - def build_docs_search(self): - """ - Build search data with separate build. - - Unless the project has the feature to allow building the JSON search - artifacts in the html build step. - """ - build_json_in_html_builder = self.project.has_feature( - Feature.BUILD_JSON_ARTIFACTS_WITH_HTML, - ) - if self.build_search and build_json_in_html_builder: - # Already built in the html step - return True - if self.build_search and self.project.is_type_sphinx: - return self.build_docs_class('sphinx_search') - return False - def build_docs_localmedia(self): """Get local media files with separate build.""" if 'htmlzip' not in self.config.formats: diff --git a/readthedocs/rtd_tests/tests/test_doc_builder.py b/readthedocs/rtd_tests/tests/test_doc_builder.py index c2b4e95ec..34c7acfc8 100644 --- a/readthedocs/rtd_tests/tests/test_doc_builder.py +++ b/readthedocs/rtd_tests/tests/test_doc_builder.py @@ -16,7 +16,7 @@ from mock import patch from readthedocs.builds.models import Version from readthedocs.doc_builder.backends.mkdocs import BaseMkdocs, MkdocsHTML -from readthedocs.doc_builder.backends.sphinx import BaseSphinx, SearchBuilder +from readthedocs.doc_builder.backends.sphinx import BaseSphinx from readthedocs.projects.exceptions import ProjectConfigurationError from readthedocs.projects.models import Project @@ -77,45 +77,6 @@ class SphinxBuilderTest(TestCase): self.assertEqual(gf.read(), ef.read()) -class SphinxSearchBuilderTest(TestCase): - - fixtures = ['test_data'] - - def setUp(self): - self.project = Project.objects.get(slug='pip') - self.version = self.project.versions.first() - - build_env = namedtuple('project', 'version') - build_env.project = self.project - build_env.version = self.version - - self.searchbuilder = SearchBuilder(build_env=build_env, python_env=None) - - def test_ignore_patterns(self): - src = tempfile.mkdtemp() - src_static = os.path.join(src, '_static/') - src_other = os.path.join(src, 'other/') - os.mkdir(src_static) - os.mkdir(src_other) - - dest = tempfile.mkdtemp() - dest_static = os.path.join(dest, '_static/') - dest_other = os.path.join(dest, 'other/') - - self.searchbuilder.old_artifact_path = src - self.searchbuilder.target = dest - - # There is a _static/ dir in src/ but not in dest/ - self.assertTrue(os.path.exists(src_static)) - self.assertFalse(os.path.exists(dest_static)) - - self.searchbuilder.move() - - # There is a dest/other/ but not a dest/_static - self.assertFalse(os.path.exists(dest_static)) - self.assertTrue(os.path.exists(dest_other)) - - @override_settings(PRODUCTION_DOMAIN='readthedocs.org') class MkdocsBuilderTest(TestCase):