Auto-generate conf.py compatible with Py2 and Py3 (#3745)

* Test to call BaseSphinx.append_conf() without a conf.py

* Fix open.write incompatibility between Py2 and Py3

* Small typo

* Small lint error

* Add docstring and mark test as failed when Exception raised

* Spaces cleanup in conf.py.conf (template)

* Improve test to check content of generated conf.py

* Remove unused import

* Increase in one the number of imported files because of new conf.py

* Fix test to return a valid temporal file as conf.py

readthedocs/projects/utils.py

@@ -10,7 +10,7 @@ import traceback
 
 import redis
 import six
-from builtins import object
+from builtins import object, open
 from django.conf import settings
 from django.core.cache import cache
 from httplib2 import Http
@@ -121,8 +121,9 @@ def safe_write(filename, contents):
     dirname = os.path.dirname(filename)
     if not os.path.exists(dirname):
         os.makedirs(dirname)
-    with open(filename, 'w') as fh:
-        fh.write(contents.encode('utf-8', 'ignore'))
+
+    with open(filename, 'w', encoding='utf-8', errors='ignore') as fh:
+        fh.write(contents)
         fh.close()
 
 

readthedocs/rtd_tests/files/conf.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import division, print_function, unicode_literals
+
+from datetime import datetime
+
+from recommonmark.parser import CommonMarkParser
+
+extensions = []
+templates_path = ['/tmp/sphinx-template-dir', 'templates', '_templates', '.templates']
+source_suffix = ['.rst', '.md']
+source_parsers = {
+            '.md': CommonMarkParser,
+        }
+master_doc = 'index'
+project = u'Pip'
+copyright = str(datetime.now().year)
+version = '0.8.1'
+release = '0.8.1'
+exclude_patterns = ['_build']
+pygments_style = 'sphinx'
+htmlhelp_basename = 'pip'
+html_theme = 'sphinx_rtd_theme'
+file_insertion_enabled = False
+latex_documents = [
+  ('index', 'pip.tex', u'Pip Documentation',
+   u'', 'manual'),
+]

readthedocs/rtd_tests/tests/test_doc_builder.py

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+from __future__ import (
+    absolute_import, division, print_function, unicode_literals)
+
+from collections import namedtuple
+import os
+import tempfile
+
+from django.test import TestCase
+from mock import patch, Mock
+import pytest
+
+from readthedocs.doc_builder.backends.sphinx import BaseSphinx
+from readthedocs.projects.exceptions import ProjectConfigurationError
+from readthedocs.projects.models import Project
+
+
+class SphinxBuilderTest(TestCase):
+
+    fixtures = ['test_data']
+
+    def setUp(self):
+        self.project = Project.objects.get(slug='pip')
+        self.version = self.project.versions.first()
+
+        build_env = namedtuple('project', 'version')
+        build_env.project = self.project
+        build_env.version = self.version
+
+        BaseSphinx.type = 'base'
+        BaseSphinx.sphinx_build_dir = tempfile.mkdtemp()
+        self.base_sphinx = BaseSphinx(build_env=build_env, python_env=None)
+
+    @patch(
+        'readthedocs.doc_builder.backends.sphinx.SPHINX_TEMPLATE_DIR',
+        '/tmp/sphinx-template-dir',
+    )
+    @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.docs_dir')
+    @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.create_index')
+    @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.get_config_params')
+    @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.run')
+    @patch('readthedocs.builds.models.Version.get_conf_py_path')
+    @patch('readthedocs.builds.models.Project.conf_file')
+    def test_create_conf_py(self, conf_file, get_conf_py_path, _, get_config_params, create_index, docs_dir):
+        """
+        Test for a project without ``conf.py`` file.
+
+        When this happen, the ``get_conf_py_path`` raises a
+        ``ProjectConfigurationError`` which is captured by our own code and
+        generates a conf.py file based using our own template.
+
+        This template should be properly rendered in Python2 and Python3 without
+        any kind of exception raised by ``append_conf`` (we were originally
+        having a ``TypeError`` because of an encoding problem in Python3)
+        """
+        docs_dir.return_value = tempfile.mkdtemp()
+        create_index.return_value = 'README.rst'
+        get_config_params.return_value = {}
+        get_conf_py_path.side_effect = ProjectConfigurationError
+        conf_file.return_value = tempfile.mktemp()
+        try:
+            self.base_sphinx.append_conf()
+        except Exception:
+            pytest.fail('Exception was generated when append_conf called.')
+
+        # Check the content generated by our method is the same than what we
+        # expects from a pre-generated file
+        generated_conf_py = os.path.join(self.base_sphinx.docs_dir(), 'conf.py')
+        expected_conf_py = os.path.join(os.path.dirname(__file__), '..', 'files', 'conf.py')
+        with open(generated_conf_py) as gf, open(expected_conf_py) as ef:
+            self.assertEqual(gf.read(), ef.read())

readthedocs/rtd_tests/tests/test_imported_file.py

@@ -19,9 +19,9 @@ class ImportedFileTests(TestCase):
         test_dir = os.path.join(base_dir, 'files')
         self.assertEqual(ImportedFile.objects.count(), 0)
         _manage_imported_files(self.version, test_dir, 'commit01')
-        self.assertEqual(ImportedFile.objects.count(), 2)
+        self.assertEqual(ImportedFile.objects.count(), 3)
         _manage_imported_files(self.version, test_dir, 'commit01')
-        self.assertEqual(ImportedFile.objects.count(), 2)
+        self.assertEqual(ImportedFile.objects.count(), 3)
 
     def test_update_commit(self):
         test_dir = os.path.join(base_dir, 'files')
@@ -47,4 +47,4 @@ class ImportedFileTests(TestCase):
         _manage_imported_files(self.version, test_dir, 'commit02')
         self.assertNotEqual(ImportedFile.objects.get(name='test.html').md5, 'c7532f22a052d716f7b2310fb52ad981')
 
-        self.assertEqual(ImportedFile.objects.count(), 2)
+        self.assertEqual(ImportedFile.objects.count(), 3)

readthedocs/templates/sphinx/conf.py.conf

@@ -8,9 +8,9 @@ from recommonmark.parser import CommonMarkParser
 
 extensions = []
 templates_path = ['{{ template_dir }}', 'templates', '_templates', '.templates']
-source_suffix = ['.rst', '.md']		
-source_parsers = {		
-            '.md': CommonMarkParser,		
+source_suffix = ['.rst', '.md']
+source_parsers = {
+            '.md': CommonMarkParser,
         }
 master_doc = 'index'
 project = u'{{ project.name }}'