EbookFoundation
/
readthedocs.org
mirror of https://github.com/EbookFoundation/readthedocs.org.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add a bunch of docstrings to tasks and doc_builder.

rtd2
Eric Holscher 2011-06-19 21:11:15 -07:00
parent 616ea0e8bc
commit 43a449bc37
6 changed files with 85 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docs/api/index.rst
View File

@ -8,6 +8,7 @@ code.
bookmarks
builds
doc_builder
core
projects
watching

5
docs/builds.rst
View File

@ -22,3 +22,8 @@ When RTD builds your project, it sets the ``READTHEDOCS`` environment variable t
html_theme = 'default'
else:
html_theme = 'nature'
Writing your own builder
========================
The documentation build system in RTD is made plugable, so that you can build out your own backend. If you have a documentation format that isn't currently supported, you can add support by contributing a backend.

10
readthedocs/core/management/commands/update_repos.py
View File

@ -23,7 +23,7 @@ class Command(BaseCommand):
),
make_option('-t',
action='store_true',
dest='touch',
dest='force',
default=False,
help='Touch the files'
),
@ -37,7 +37,7 @@ class Command(BaseCommand):
def handle(self, *args, **options):
make_pdf = options['pdf']
record = options['record']
touch = options['touch']
force = options['force']
version = options['version']
if len(args):
for slug in args:
@ -61,7 +61,7 @@ class Command(BaseCommand):
else:
p = Project.objects.get(slug=slug)
print "Building %s" % p
tasks.update_docs(pk=p.pk, pdf=make_pdf, touch=touch)
tasks.update_docs(pk=p.pk, pdf=make_pdf, force=force)
else:
if version == "all":
print "Updating all versions"
@ -70,10 +70,10 @@ class Command(BaseCommand):
tasks.update_docs(pk=version.project_id,
pdf=make_pdf,
record=record,
touch=touch,
force=force,
version_pk=version.pk)
else:
print "Updating all docs"
tasks.update_docs_pull(pdf=make_pdf,
record=record,
touch=touch)
force=force)

8
readthedocs/core/views.py
View File

@ -53,7 +53,7 @@ def github_build(request):
ghetto_url = url.replace('http://', '').replace('https://', '')
try:
project = Project.objects.filter(repo__contains=ghetto_url)[0]
update_docs.delay(pk=project.pk, touch=True)
update_docs.delay(pk=project.pk, force=True)
return HttpResponse('Build Started')
except:
mail_admins('Build Failure', '%s failed to build via github' % name)
@ -71,7 +71,7 @@ def bitbucket_build(request):
url = "%s%s" % ("bitbucket.org", rep['absolute_url'])
try:
project = Project.objects.filter(repo__contains=url)[0]
update_docs.delay(pk=project.pk, touch=True)
update_docs.delay(pk=project.pk, force=True)
return HttpResponse('Build Started')
except:
mail_admins('Build Failure', '%s failed to build via bitbucket' % name)
@ -89,9 +89,9 @@ def generic_build(request, pk):
slug = request.POST.get('version_slug', None)
if slug:
version = project.versions.get(slug=slug)
update_docs.delay(pk=pk, version_pk=version.pk, touch=True)
update_docs.delay(pk=pk, version_pk=version.pk, force=True)
else:
update_docs.delay(pk=pk, touch=True)
update_docs.delay(pk=pk, force=True)
return HttpResponse('Build Started')
return render_to_response('post_commit.html', context,
context_instance=RequestContext(request))

30
readthedocs/doc_builder/base.py
View File

@ -12,15 +12,41 @@ def restoring_chdir(fn):
class BaseBuilder(object):
"""
The Base for all Builders. Defines the API for subclasses.
"""
@restoring_chdir
def touch(self, version):
print "Touching files"
def force(self, version):
"""
An optional step to force a build even when nothing has changed.
"""
print "Forcing a build by touching files"
os.chdir(version.project.conf_dir(version.slug))
os.system('touch * && touch */*')
def clean(self, version):
"""
Clean up the version so it's ready for usage.
This is used to add RTD specific stuff to Sphinx, and to
implement whitelists on projects as well.
It is guaranteed to be called before your project is built.
"""
raise NotImplementedError
def build(self, version):
"""
Do the actual building of the documentation.
"""
raise NotImplementedError
def move(self, version):
"""
Move the documentation from it's generated place to its final home.
This needs to understand both a single server dev environment,
as well as a multi-server environment.
"""
raise NotImplementedError

52
readthedocs/projects/tasks.py
View File

@ -35,14 +35,24 @@ ghetto_hack = re.compile(
@task
def remove_dir(path):
"""
Remove a directory on the build/celery server.
This is mainly a wrapper around shutil.rmtree so that app servers
can kill things on the build server.
"""
print "Removing %s" % path
shutil.rmtree(path)
@task
def update_docs(pk, record=True, pdf=True, man=True, epub=True, version_pk=None, touch=False):
def update_docs(pk, record=True, pdf=True, man=True, epub=True, version_pk=None, force=False):
"""
A Celery task that updates the documentation for a project.
The main entry point for updating documentation.
It handles all of the logic around whether a project is imported or we created it.
Then it will build the html docs and other requested parts.
It also handles clearing the varnish cache.
"""
###
@ -88,7 +98,7 @@ def update_docs(pk, record=True, pdf=True, man=True, epub=True, version_pk=None,
# kick off a build
(ret, out, err) = build_docs(project=project, version=version,
pdf=pdf, man=man, epub=epub,
record=record, touch=touch)
record=record, force=force)
if not 'no targets are out of date.' in out:
if ret == 0:
print "HTML Build OK"
@ -213,7 +223,8 @@ def update_imported_docs(project, version):
def scrape_conf_file(version):
"""Locate the given project's ``conf.py`` file and extract important
"""
Locate the given project's ``conf.py`` file and extract important
settings, including copyright, theme, source suffix and version.
"""
#This is where we actually find the conf.py, so we can't use
@ -250,6 +261,9 @@ def scrape_conf_file(version):
def update_created_docs(project):
"""
Handle generating the docs for projects hosted on RTD.
"""
# grab the root path for the generated docs to live at
path = project.doc_path
@ -269,16 +283,16 @@ def update_created_docs(project):
file.write_to_disk()
def build_docs(project, version, pdf, man, epub, record, touch):
def build_docs(project, version, pdf, man, epub, record, force):
"""
A helper function for the celery task to do the actual doc building.
This handles the actual building of the documentation and DB records
"""
if not project.conf_file(version.slug):
return ('', 'Conf file not found.', -1)
html_builder = builder_loading.get(project.documentation_type)()
if touch:
html_builder.touch(version)
if force:
html_builder.force(version)
html_builder.clean(version)
html_output = html_builder.build(version)
successful = (html_output[0] == 0)
@ -310,6 +324,7 @@ def build_docs(project, version, pdf, man, epub, record, touch):
return html_output
def move_docs(project, version):
#XXX:eh: This should be moved into the sphinx builder.
if project.full_build_path(version.slug):
target = project.rtd_build_path(version.slug)
if getattr(settings, "MULTIPLE_APP_SERVERS", None):
@ -322,6 +337,9 @@ def move_docs(project, version):
print "Not moving docs, because the build dir is unknown."
def copy_to_app_servers(full_build_path, target, mkdir=True):
"""
A helper to copy a directory across app servers
"""
print "Copying %s to %s" % (full_build_path, target)
for server in settings.MULTIPLE_APP_SERVERS:
os.system("ssh %s@%s mkdir -p %s" % (getpass.getuser(), server, target))
@ -330,6 +348,9 @@ def copy_to_app_servers(full_build_path, target, mkdir=True):
print "COPY ERROR to app servers."
def copy_file_to_app_servers(from_file, to_file):
"""
A helper to copy a single file across app servers
"""
print "Copying %s to %s" % (from_file, to_file)
to_path = '/'.join(to_file.split('/')[0:-1])
for server in settings.MULTIPLE_APP_SERVERS:
@ -340,6 +361,12 @@ def copy_file_to_app_servers(from_file, to_file):
def fileify(version):
"""
Create ImportedFile objects for all of a version's files.
This is a prereq for indexing the docs for search.
"""
project = version.project
path = project.rtd_build_path(version.slug)
if path:
@ -354,10 +381,15 @@ def fileify(version):
#@periodic_task(run_every=crontab(hour="2", minute="10", day_of_week="*"))
def update_docs_pull(record=False, pdf=False, man=False, touch=False):
def update_docs_pull(record=False, pdf=False, man=False, force=False):
"""
A high-level interface that will update all of the projects.
This is mainly used from a cronjob or management command.
"""
for project in Project.objects.live():
try:
update_docs(pk=project.pk, record=record, pdf=pdf, man=man, touch=touch)
update_docs(pk=project.pk, record=record, pdf=pdf, man=man, force=force)
except:
print "failed"