Add a bunch of docstrings to tasks and doc_builder.
parent
616ea0e8bc
commit
43a449bc37
|
@ -8,6 +8,7 @@ code.
|
||||||
|
|
||||||
bookmarks
|
bookmarks
|
||||||
builds
|
builds
|
||||||
|
doc_builder
|
||||||
core
|
core
|
||||||
projects
|
projects
|
||||||
watching
|
watching
|
||||||
|
|
|
@ -22,3 +22,8 @@ When RTD builds your project, it sets the ``READTHEDOCS`` environment variable t
|
||||||
html_theme = 'default'
|
html_theme = 'default'
|
||||||
else:
|
else:
|
||||||
html_theme = 'nature'
|
html_theme = 'nature'
|
||||||
|
|
||||||
|
Writing your own builder
|
||||||
|
========================
|
||||||
|
|
||||||
|
The documentation build system in RTD is made plugable, so that you can build out your own backend. If you have a documentation format that isn't currently supported, you can add support by contributing a backend.
|
||||||
|
|
|
@ -23,7 +23,7 @@ class Command(BaseCommand):
|
||||||
),
|
),
|
||||||
make_option('-t',
|
make_option('-t',
|
||||||
action='store_true',
|
action='store_true',
|
||||||
dest='touch',
|
dest='force',
|
||||||
default=False,
|
default=False,
|
||||||
help='Touch the files'
|
help='Touch the files'
|
||||||
),
|
),
|
||||||
|
@ -37,7 +37,7 @@ class Command(BaseCommand):
|
||||||
def handle(self, *args, **options):
|
def handle(self, *args, **options):
|
||||||
make_pdf = options['pdf']
|
make_pdf = options['pdf']
|
||||||
record = options['record']
|
record = options['record']
|
||||||
touch = options['touch']
|
force = options['force']
|
||||||
version = options['version']
|
version = options['version']
|
||||||
if len(args):
|
if len(args):
|
||||||
for slug in args:
|
for slug in args:
|
||||||
|
@ -61,7 +61,7 @@ class Command(BaseCommand):
|
||||||
else:
|
else:
|
||||||
p = Project.objects.get(slug=slug)
|
p = Project.objects.get(slug=slug)
|
||||||
print "Building %s" % p
|
print "Building %s" % p
|
||||||
tasks.update_docs(pk=p.pk, pdf=make_pdf, touch=touch)
|
tasks.update_docs(pk=p.pk, pdf=make_pdf, force=force)
|
||||||
else:
|
else:
|
||||||
if version == "all":
|
if version == "all":
|
||||||
print "Updating all versions"
|
print "Updating all versions"
|
||||||
|
@ -70,10 +70,10 @@ class Command(BaseCommand):
|
||||||
tasks.update_docs(pk=version.project_id,
|
tasks.update_docs(pk=version.project_id,
|
||||||
pdf=make_pdf,
|
pdf=make_pdf,
|
||||||
record=record,
|
record=record,
|
||||||
touch=touch,
|
force=force,
|
||||||
version_pk=version.pk)
|
version_pk=version.pk)
|
||||||
else:
|
else:
|
||||||
print "Updating all docs"
|
print "Updating all docs"
|
||||||
tasks.update_docs_pull(pdf=make_pdf,
|
tasks.update_docs_pull(pdf=make_pdf,
|
||||||
record=record,
|
record=record,
|
||||||
touch=touch)
|
force=force)
|
||||||
|
|
|
@ -53,7 +53,7 @@ def github_build(request):
|
||||||
ghetto_url = url.replace('http://', '').replace('https://', '')
|
ghetto_url = url.replace('http://', '').replace('https://', '')
|
||||||
try:
|
try:
|
||||||
project = Project.objects.filter(repo__contains=ghetto_url)[0]
|
project = Project.objects.filter(repo__contains=ghetto_url)[0]
|
||||||
update_docs.delay(pk=project.pk, touch=True)
|
update_docs.delay(pk=project.pk, force=True)
|
||||||
return HttpResponse('Build Started')
|
return HttpResponse('Build Started')
|
||||||
except:
|
except:
|
||||||
mail_admins('Build Failure', '%s failed to build via github' % name)
|
mail_admins('Build Failure', '%s failed to build via github' % name)
|
||||||
|
@ -71,7 +71,7 @@ def bitbucket_build(request):
|
||||||
url = "%s%s" % ("bitbucket.org", rep['absolute_url'])
|
url = "%s%s" % ("bitbucket.org", rep['absolute_url'])
|
||||||
try:
|
try:
|
||||||
project = Project.objects.filter(repo__contains=url)[0]
|
project = Project.objects.filter(repo__contains=url)[0]
|
||||||
update_docs.delay(pk=project.pk, touch=True)
|
update_docs.delay(pk=project.pk, force=True)
|
||||||
return HttpResponse('Build Started')
|
return HttpResponse('Build Started')
|
||||||
except:
|
except:
|
||||||
mail_admins('Build Failure', '%s failed to build via bitbucket' % name)
|
mail_admins('Build Failure', '%s failed to build via bitbucket' % name)
|
||||||
|
@ -89,9 +89,9 @@ def generic_build(request, pk):
|
||||||
slug = request.POST.get('version_slug', None)
|
slug = request.POST.get('version_slug', None)
|
||||||
if slug:
|
if slug:
|
||||||
version = project.versions.get(slug=slug)
|
version = project.versions.get(slug=slug)
|
||||||
update_docs.delay(pk=pk, version_pk=version.pk, touch=True)
|
update_docs.delay(pk=pk, version_pk=version.pk, force=True)
|
||||||
else:
|
else:
|
||||||
update_docs.delay(pk=pk, touch=True)
|
update_docs.delay(pk=pk, force=True)
|
||||||
return HttpResponse('Build Started')
|
return HttpResponse('Build Started')
|
||||||
return render_to_response('post_commit.html', context,
|
return render_to_response('post_commit.html', context,
|
||||||
context_instance=RequestContext(request))
|
context_instance=RequestContext(request))
|
||||||
|
|
|
@ -12,15 +12,41 @@ def restoring_chdir(fn):
|
||||||
|
|
||||||
|
|
||||||
class BaseBuilder(object):
|
class BaseBuilder(object):
|
||||||
|
"""
|
||||||
|
The Base for all Builders. Defines the API for subclasses.
|
||||||
|
"""
|
||||||
|
|
||||||
@restoring_chdir
|
@restoring_chdir
|
||||||
def touch(self, version):
|
def force(self, version):
|
||||||
print "Touching files"
|
"""
|
||||||
|
An optional step to force a build even when nothing has changed.
|
||||||
|
"""
|
||||||
|
print "Forcing a build by touching files"
|
||||||
os.chdir(version.project.conf_dir(version.slug))
|
os.chdir(version.project.conf_dir(version.slug))
|
||||||
os.system('touch * && touch */*')
|
os.system('touch * && touch */*')
|
||||||
|
|
||||||
def clean(self, version):
|
def clean(self, version):
|
||||||
|
"""
|
||||||
|
Clean up the version so it's ready for usage.
|
||||||
|
|
||||||
|
This is used to add RTD specific stuff to Sphinx, and to
|
||||||
|
implement whitelists on projects as well.
|
||||||
|
|
||||||
|
It is guaranteed to be called before your project is built.
|
||||||
|
"""
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
def build(self, version):
|
def build(self, version):
|
||||||
|
"""
|
||||||
|
Do the actual building of the documentation.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def move(self, version):
|
||||||
|
"""
|
||||||
|
Move the documentation from it's generated place to its final home.
|
||||||
|
|
||||||
|
This needs to understand both a single server dev environment,
|
||||||
|
as well as a multi-server environment.
|
||||||
|
"""
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
|
@ -35,14 +35,24 @@ ghetto_hack = re.compile(
|
||||||
|
|
||||||
@task
|
@task
|
||||||
def remove_dir(path):
|
def remove_dir(path):
|
||||||
|
"""
|
||||||
|
Remove a directory on the build/celery server.
|
||||||
|
|
||||||
|
This is mainly a wrapper around shutil.rmtree so that app servers
|
||||||
|
can kill things on the build server.
|
||||||
|
"""
|
||||||
print "Removing %s" % path
|
print "Removing %s" % path
|
||||||
shutil.rmtree(path)
|
shutil.rmtree(path)
|
||||||
|
|
||||||
|
|
||||||
@task
|
@task
|
||||||
def update_docs(pk, record=True, pdf=True, man=True, epub=True, version_pk=None, touch=False):
|
def update_docs(pk, record=True, pdf=True, man=True, epub=True, version_pk=None, force=False):
|
||||||
"""
|
"""
|
||||||
A Celery task that updates the documentation for a project.
|
The main entry point for updating documentation.
|
||||||
|
|
||||||
|
It handles all of the logic around whether a project is imported or we created it.
|
||||||
|
Then it will build the html docs and other requested parts.
|
||||||
|
It also handles clearing the varnish cache.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
###
|
###
|
||||||
|
@ -88,7 +98,7 @@ def update_docs(pk, record=True, pdf=True, man=True, epub=True, version_pk=None,
|
||||||
# kick off a build
|
# kick off a build
|
||||||
(ret, out, err) = build_docs(project=project, version=version,
|
(ret, out, err) = build_docs(project=project, version=version,
|
||||||
pdf=pdf, man=man, epub=epub,
|
pdf=pdf, man=man, epub=epub,
|
||||||
record=record, touch=touch)
|
record=record, force=force)
|
||||||
if not 'no targets are out of date.' in out:
|
if not 'no targets are out of date.' in out:
|
||||||
if ret == 0:
|
if ret == 0:
|
||||||
print "HTML Build OK"
|
print "HTML Build OK"
|
||||||
|
@ -213,7 +223,8 @@ def update_imported_docs(project, version):
|
||||||
|
|
||||||
|
|
||||||
def scrape_conf_file(version):
|
def scrape_conf_file(version):
|
||||||
"""Locate the given project's ``conf.py`` file and extract important
|
"""
|
||||||
|
Locate the given project's ``conf.py`` file and extract important
|
||||||
settings, including copyright, theme, source suffix and version.
|
settings, including copyright, theme, source suffix and version.
|
||||||
"""
|
"""
|
||||||
#This is where we actually find the conf.py, so we can't use
|
#This is where we actually find the conf.py, so we can't use
|
||||||
|
@ -250,6 +261,9 @@ def scrape_conf_file(version):
|
||||||
|
|
||||||
|
|
||||||
def update_created_docs(project):
|
def update_created_docs(project):
|
||||||
|
"""
|
||||||
|
Handle generating the docs for projects hosted on RTD.
|
||||||
|
"""
|
||||||
# grab the root path for the generated docs to live at
|
# grab the root path for the generated docs to live at
|
||||||
path = project.doc_path
|
path = project.doc_path
|
||||||
|
|
||||||
|
@ -269,16 +283,16 @@ def update_created_docs(project):
|
||||||
file.write_to_disk()
|
file.write_to_disk()
|
||||||
|
|
||||||
|
|
||||||
def build_docs(project, version, pdf, man, epub, record, touch):
|
def build_docs(project, version, pdf, man, epub, record, force):
|
||||||
"""
|
"""
|
||||||
A helper function for the celery task to do the actual doc building.
|
This handles the actual building of the documentation and DB records
|
||||||
"""
|
"""
|
||||||
if not project.conf_file(version.slug):
|
if not project.conf_file(version.slug):
|
||||||
return ('', 'Conf file not found.', -1)
|
return ('', 'Conf file not found.', -1)
|
||||||
|
|
||||||
html_builder = builder_loading.get(project.documentation_type)()
|
html_builder = builder_loading.get(project.documentation_type)()
|
||||||
if touch:
|
if force:
|
||||||
html_builder.touch(version)
|
html_builder.force(version)
|
||||||
html_builder.clean(version)
|
html_builder.clean(version)
|
||||||
html_output = html_builder.build(version)
|
html_output = html_builder.build(version)
|
||||||
successful = (html_output[0] == 0)
|
successful = (html_output[0] == 0)
|
||||||
|
@ -310,6 +324,7 @@ def build_docs(project, version, pdf, man, epub, record, touch):
|
||||||
return html_output
|
return html_output
|
||||||
|
|
||||||
def move_docs(project, version):
|
def move_docs(project, version):
|
||||||
|
#XXX:eh: This should be moved into the sphinx builder.
|
||||||
if project.full_build_path(version.slug):
|
if project.full_build_path(version.slug):
|
||||||
target = project.rtd_build_path(version.slug)
|
target = project.rtd_build_path(version.slug)
|
||||||
if getattr(settings, "MULTIPLE_APP_SERVERS", None):
|
if getattr(settings, "MULTIPLE_APP_SERVERS", None):
|
||||||
|
@ -322,6 +337,9 @@ def move_docs(project, version):
|
||||||
print "Not moving docs, because the build dir is unknown."
|
print "Not moving docs, because the build dir is unknown."
|
||||||
|
|
||||||
def copy_to_app_servers(full_build_path, target, mkdir=True):
|
def copy_to_app_servers(full_build_path, target, mkdir=True):
|
||||||
|
"""
|
||||||
|
A helper to copy a directory across app servers
|
||||||
|
"""
|
||||||
print "Copying %s to %s" % (full_build_path, target)
|
print "Copying %s to %s" % (full_build_path, target)
|
||||||
for server in settings.MULTIPLE_APP_SERVERS:
|
for server in settings.MULTIPLE_APP_SERVERS:
|
||||||
os.system("ssh %s@%s mkdir -p %s" % (getpass.getuser(), server, target))
|
os.system("ssh %s@%s mkdir -p %s" % (getpass.getuser(), server, target))
|
||||||
|
@ -330,6 +348,9 @@ def copy_to_app_servers(full_build_path, target, mkdir=True):
|
||||||
print "COPY ERROR to app servers."
|
print "COPY ERROR to app servers."
|
||||||
|
|
||||||
def copy_file_to_app_servers(from_file, to_file):
|
def copy_file_to_app_servers(from_file, to_file):
|
||||||
|
"""
|
||||||
|
A helper to copy a single file across app servers
|
||||||
|
"""
|
||||||
print "Copying %s to %s" % (from_file, to_file)
|
print "Copying %s to %s" % (from_file, to_file)
|
||||||
to_path = '/'.join(to_file.split('/')[0:-1])
|
to_path = '/'.join(to_file.split('/')[0:-1])
|
||||||
for server in settings.MULTIPLE_APP_SERVERS:
|
for server in settings.MULTIPLE_APP_SERVERS:
|
||||||
|
@ -340,6 +361,12 @@ def copy_file_to_app_servers(from_file, to_file):
|
||||||
|
|
||||||
|
|
||||||
def fileify(version):
|
def fileify(version):
|
||||||
|
"""
|
||||||
|
Create ImportedFile objects for all of a version's files.
|
||||||
|
|
||||||
|
This is a prereq for indexing the docs for search.
|
||||||
|
"""
|
||||||
|
|
||||||
project = version.project
|
project = version.project
|
||||||
path = project.rtd_build_path(version.slug)
|
path = project.rtd_build_path(version.slug)
|
||||||
if path:
|
if path:
|
||||||
|
@ -354,10 +381,15 @@ def fileify(version):
|
||||||
|
|
||||||
|
|
||||||
#@periodic_task(run_every=crontab(hour="2", minute="10", day_of_week="*"))
|
#@periodic_task(run_every=crontab(hour="2", minute="10", day_of_week="*"))
|
||||||
def update_docs_pull(record=False, pdf=False, man=False, touch=False):
|
def update_docs_pull(record=False, pdf=False, man=False, force=False):
|
||||||
|
"""
|
||||||
|
A high-level interface that will update all of the projects.
|
||||||
|
|
||||||
|
This is mainly used from a cronjob or management command.
|
||||||
|
"""
|
||||||
for project in Project.objects.live():
|
for project in Project.objects.live():
|
||||||
try:
|
try:
|
||||||
update_docs(pk=project.pk, record=record, pdf=pdf, man=man, touch=touch)
|
update_docs(pk=project.pk, record=record, pdf=pdf, man=man, force=force)
|
||||||
except:
|
except:
|
||||||
print "failed"
|
print "failed"
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue