"""Core views, including the main homepage, post-commit build hook, documentation and header rendering, and server errors. """ from django.contrib.auth.models import User from django.core.urlresolvers import reverse from django.conf import settings from django.http import HttpResponse, HttpResponseRedirect, Http404, HttpResponseNotFound from django.shortcuts import render_to_response, get_object_or_404, redirect from django.template import RequestContext from django.views.decorators.csrf import csrf_view_exempt from django.views.static import serve from django.views.generic import TemplateView from haystack.query import EmptySearchQuerySet from haystack.query import SearchQuerySet from celery.task.control import inspect from builds.models import Build from builds.models import Version from core.forms import FacetedSearchForm from projects.models import Project, ImportedFile, ProjectRelationship from projects.tasks import update_docs, remove_dir import json import mimetypes import os import logging import redis import re log = logging.getLogger(__name__) pc_log = logging.getLogger(__name__+'.post_commit') class NoProjectException(Exception): pass def homepage(request): latest = (Project.objects.public(request.user) .order_by('-modified_date')[:10]) featured = Project.objects.filter(featured=True) return render_to_response('homepage.html', {'project_list': latest, 'featured_list': featured}, context_instance=RequestContext(request)) def random_page(request, project=None): imp_file = ImportedFile.objects.order_by('?') if project: return HttpResponseRedirect((imp_file.filter(project__slug=project)[0] .get_absolute_url())) return HttpResponseRedirect(imp_file[0].get_absolute_url()) def queue_depth(request): r = redis.Redis(**settings.REDIS) return HttpResponse(r.llen('celery')) def queue_info(request): i = inspect() active_pks = [] reserved_pks = [] resp = "" active = i.active() if active: try: for obj in active['build']: kwargs = eval(obj['kwargs']) active_pks.append(str(kwargs['pk'])) active_resp = "Active: %s " % " ".join(active_pks) resp += active_resp except Exception, e: resp += str(e) reserved = i.reserved() if reserved: try: for obj in reserved['build']: kwrags = eval(obj['kwargs']) reserved_pks.append(str(kwargs['pk'])) reserved_resp = " | Reserved: %s" % " ".join(reserved_pks) resp += reserved_resp except Exception, e: resp += str(e) return HttpResponse(resp) def live_builds(request): builds = Build.objects.filter(state='building')[:5] WEBSOCKET_HOST = getattr(settings, 'WEBSOCKET_HOST', 'localhost:8088') count = builds.count() percent = 100 if count > 1: percent = 100 / count return render_to_response('all_builds.html', {'builds': builds, 'build_percent': percent, 'WEBSOCKET_HOST': WEBSOCKET_HOST}, context_instance=RequestContext(request)) @csrf_view_exempt def wipe_version(request, project_slug, version_slug): version = get_object_or_404(Version, project__slug=project_slug, slug=version_slug) if request.user not in version.project.users.all(): raise Http404("You must own this project to wipe it.") del_dirs = [version.project.checkout_path(version.slug), version.project.venv_path(version.slug)] for del_dir in del_dirs: remove_dir.delay(del_dir) return render_to_response('wipe_version.html', {'del_dir': del_dir}, context_instance=RequestContext(request)) def _build_version(project, slug, already_built=()): default = project.default_branch or (project.vcs_repo().fallback_branch) if slug == default and slug not in already_built: # short circuit versions that are default # these will build at "latest", and thus won't be # active latest_version = project.versions.get(slug='latest') update_docs.delay(pk=project.pk, version_pk=latest_version.pk, force=True) pc_log.info(("(Version build) Building %s:%s" % (project.slug, latest_version.slug))) if project.versions.exclude(active=False).filter(slug=slug).exists(): # Handle the case where we want to build the custom branch too slug_version = project.versions.get(slug=slug) update_docs.delay(pk=project.pk, version_pk=slug_version.pk, force=True) pc_log.info(("(Version build) Building %s:%s" % (project.slug, slug_version.slug))) return "latest" elif project.versions.exclude(active=True).filter(slug=slug).exists(): pc_log.info(("(Version build) Not Building %s"% slug)) return None elif slug not in already_built: version = project.versions.get(slug=slug) update_docs.delay(pk=project.pk, version_pk=version.pk, force=True) pc_log.info(("(Version build) Building %s:%s" % (project.slug, version.slug))) return slug else: pc_log.info(("(Version build) Not Building %s"% slug)) return None def _build_branches(project, branch_list): for branch in branch_list: versions = project.versions_from_branch_name(branch) to_build = set() not_building = set() for version in versions: pc_log.info(("(Branch Build) Processing %s:%s" % (project.slug, version.slug))) ret = _build_version(project, version.slug, already_built=to_build) if ret: to_build.add(ret) else: not_building.add(version.slug) return (to_build, not_building) def _build_url(url, branches): try: projects = Project.objects.filter(repo__endswith=url) if not projects.count(): projects = Project.objects.filter(repo__endswith=url + '.git') if not projects.count(): raise NoProjectException() for project in projects: (to_build, not_building) = _build_branches(project, branches) if to_build: msg = '(URL Build) Build Started: %s [%s]' % (url, ' '.join(to_build)) pc_log.info(msg) return HttpResponse(msg) else: msg = '(URL Build) Not Building: %s [%s]' % (url, ' '.join(not_building)) pc_log.info(msg) return HttpResponse(msg) except Exception, e: if e.__class__ == NoProjectException: raise msg = "(URL Build) Failed: %s:%s" % (url, e) pc_log.error(msg) return HttpResponse(msg) @csrf_view_exempt def github_build(request): """ A post-commit hook for github. """ if request.method == 'POST': obj = json.loads(request.POST['payload']) url = obj['repository']['url'] ghetto_url = url.replace('http://', '').replace('https://', '') branch = obj['ref'].replace('refs/heads/', '') pc_log.info("(Incoming Github Build) %s [%s]" % (ghetto_url, branch)) try: return _build_url(ghetto_url, [branch]) except NoProjectException: try: name = obj['repository']['name'] desc = obj['repository']['description'] homepage = obj['repository']['homepage'] repo = obj['repository']['url'] email = obj['repository']['owner']['email'] user = User.objects.get(email=email) proj = Project.objects.create( name=name, description=desc, project_url=homepage, repo=repo, ) proj.users.add(user) # Version doesn't exist yet, so use classic build method update_docs.delay(pk=proj.pk) pc_log.info("Created new project %s" % (proj)) except Exception, e: pc_log.error("Error creating new project %s: %s" % (name, e)) return HttpResponseNotFound('Repo not found') @csrf_view_exempt def bitbucket_build(request): if request.method == 'POST': payload = request.POST.get('payload') pc_log.info("(Incoming Bitbucket Build) Raw: %s" % payload) if not payload: return HttpResponseNotFound('Invalid Request') obj = json.loads(payload) rep = obj['repository'] branches = [rec.get('branch', '') for rec in obj['commits']] ghetto_url = "%s%s" % ("bitbucket.org", rep['absolute_url'].rstrip('/')) pc_log.info("(Incoming Bitbucket Build) %s [%s]" % (ghetto_url, ' '.join(branches))) pc_log.info("(Incoming Bitbucket Build) JSON: \n\n%s\n\n" % obj) try: return _build_url(ghetto_url, branches) except NoProjectException: pc_log.error("(Incoming Bitbucket Build) Repo not found: %s" % ghetto_url) return HttpResponseNotFound('Repo not found: %s' % ghetto_url) @csrf_view_exempt def generic_build(request, pk=None): try: project = Project.objects.get(pk=pk) # Allow slugs too except (Project.DoesNotExist, ValueError): project = Project.objects.get(slug=pk) if request.method == 'POST': slug = request.POST.get('version_slug', None) if slug: pc_log.info("(Incoming Generic Build) %s [%s]" % (project.slug, slug)) _build_version(project, slug) else: pc_log.info("(Incoming Generic Build) %s [%s]" % (project.slug, 'latest')) update_docs.delay(pk=pk, force=True) return redirect('builds_project_list', project.slug) def subproject_list(request): project_slug = request.slug proj = get_object_or_404(Project, slug=project_slug) subprojects = [rel.child for rel in proj.subprojects.all()] return render_to_response( 'projects/project_list.html', {'project_list': subprojects}, context_instance=RequestContext(request) ) def subproject_serve_docs(request, project_slug, lang_slug=None, version_slug=None, filename=''): parent_slug = request.slug proj = get_object_or_404(Project, slug=project_slug) subproject_qs = ProjectRelationship.objects.filter( parent__slug=parent_slug, child__slug=project_slug) if lang_slug is None or version_slug is None: # Handle / version_slug = proj.get_default_version() url = reverse('subproject_docs_detail', kwargs={ 'project_slug': project_slug, 'version_slug': version_slug, 'lang_slug': proj.language, 'filename': filename }) return HttpResponseRedirect(url) if subproject_qs.exists(): return serve_docs(request, lang_slug, version_slug, filename, project_slug) else: log.info('Subproject lookup failed: %s:%s' % (project_slug, parent_slug)) raise Http404("Subproject does not exist") def default_docs_kwargs(request, project_slug=None): """ Return kwargs used to reverse lookup a project's default docs URL. Determining which URL to redirect to is done based on the kwargs passed to reverse(serve_docs, kwargs). This function populates kwargs for the default docs for a project, and sets appropriate keys depending on whether request is for a subdomain URL, or a non-subdomain URL. """ if project_slug: try: proj = Project.objects.get(slug=project_slug) except (Project.DoesNotExist, ValueError): # Try with underscore, for legacy try: proj = Project.objects.get(slug=project_slug.replace('-', '_')) except (Project.DoesNotExist): proj = None else: # If project_slug isn't in URL pattern, it's set in subdomain # middleware as request.slug. try: proj = Project.objects.get(slug=request.slug) except (Project.DoesNotExist, ValueError): # Try with underscore, for legacy try: proj = Project.objects.get(slug=request.slug.replace('-', '_')) except (Project.DoesNotExist): proj = None if not proj: raise Http404("Project slug not found") version_slug = proj.get_default_version() kwargs = { 'project_slug': project_slug, 'version_slug': version_slug, 'lang_slug': proj.language, 'filename': '' } # Don't include project_slug for subdomains. # That's how reverse(serve_docs, ...) differentiates subdomain # views from non-subdomain views. if project_slug is None: del kwargs['project_slug'] return kwargs def redirect_lang_slug(request, lang_slug, project_slug=None): """Redirect /en/ to /en/latest/.""" kwargs = default_docs_kwargs(request, project_slug) kwargs['lang_slug'] = lang_slug url = reverse(serve_docs, kwargs=kwargs) return HttpResponseRedirect(url) def redirect_version_slug(request, version_slug, project_slug=None): """Redirect /latest/ to /en/latest/.""" kwargs = default_docs_kwargs(request, project_slug) kwargs['version_slug'] = version_slug url = reverse(serve_docs, kwargs=kwargs) return HttpResponseRedirect(url) def redirect_project_slug(request, project_slug=None): """Redirect / to /en/latest/.""" kwargs = default_docs_kwargs(request, project_slug) url = reverse(serve_docs, kwargs=kwargs) return HttpResponseRedirect(url) def redirect_page_with_filename(request, filename, project_slug=None): """Redirect /page/file.html to /en/latest/file.html.""" kwargs = default_docs_kwargs(request, project_slug) kwargs['filename'] = filename url = reverse(serve_docs, kwargs=kwargs) return HttpResponseRedirect(url) def serve_docs(request, lang_slug, version_slug, filename, project_slug=None): if not project_slug: project_slug = request.slug try: proj = Project.objects.get(slug=project_slug) ver = Version.objects.get(project__slug=project_slug, slug=version_slug) except (Project.DoesNotExist, Version.DoesNotExist): proj = None ver = None if not proj or not ver: return server_helpful_404(request, project_slug, lang_slug, version_slug, filename) # Auth checks if ver not in proj.versions.public(request.user, proj, only_active=False): res = HttpResponse("You don't have access to this version.") res.status_code = 401 return res # Figure out actual file to serve if not filename: filename = "index.html" # This is required because we're forming the filenames outselves instead of # letting the web server do it. elif (proj.documentation_type == 'sphinx_htmldir' and "_static" not in filename and "_images" not in filename and "html" not in filename and not "inv" in filename): filename += "index.html" else: filename = filename.rstrip('/') # Use the old paths if we're on our old location. # Otherwise use the new language symlinks. # This can be removed once we have 'en' symlinks for every project. if lang_slug == proj.language: basepath = proj.rtd_build_path(version_slug) else: basepath = proj.translations_symlink_path(lang_slug) basepath = os.path.join(basepath, version_slug) # Serve file log.info('Serving %s for %s' % (filename, proj)) if not settings.DEBUG: fullpath = os.path.join(basepath, filename) mimetype, encoding = mimetypes.guess_type(fullpath) mimetype = mimetype or 'application/octet-stream' response = HttpResponse(mimetype=mimetype) if encoding: response["Content-Encoding"] = encoding try: response['X-Accel-Redirect'] = os.path.join(basepath[len(settings.SITE_ROOT):], filename) except UnicodeEncodeError: raise Http404 return response else: return serve(request, filename, basepath) def serve_single_version_docs(request, filename, project_slug=None): if not project_slug: project_slug = request.slug proj = get_object_or_404(Project, slug=project_slug) # This function only handles single version projects if not proj.single_version: raise Http404 return serve_docs(request, proj.language, proj.default_version, filename, project_slug) def server_error(request, template_name='500.html'): """ A simple 500 handler so we get media """ r = render_to_response(template_name, context_instance=RequestContext(request)) r.status_code = 500 return r def server_error_404(request, template_name='404.html'): """ A simple 404 handler so we get media """ r = render_to_response(template_name, context_instance=RequestContext(request)) r.status_code = 404 return r def server_helpful_404(request, project_slug=None, lang_slug=None, version_slug=None, filename=None, template_name='404.html'): pagename = re.sub(r'/index$', r'', re.sub(r'\.html$', r'', re.sub(r'/$', r'', filename))) suggestion = get_suggestion(project_slug, lang_slug, version_slug, pagename, request.user) r = render_to_response(template_name, {'suggestion': suggestion}, context_instance=RequestContext(request)) r.status_code = 404 return r def get_suggestion(project_slug, lang_slug, version_slug, pagename, user): """ | # | project | version | language | What to show | | 1 | 0 | 0 | 0 | Error message | | 2 | 0 | 0 | 1 | Error message (Can't happen) | | 3 | 0 | 1 | 0 | Error message (Can't happen) | | 4 | 0 | 1 | 1 | Error message (Can't happen) | | 5 | 1 | 0 | 0 | A link to top-level page of default version | | 6 | 1 | 0 | 1 | Available versions on the translation project | | 7 | 1 | 1 | 0 | Available translations of requested version | | 8 | 1 | 1 | 1 | A link to top-level page of requested version | """ suggestion = {} if project_slug: try: proj = Project.objects.get(slug=project_slug) if not lang_slug: lang_slug = proj.language try: ver = Version.objects.get(project__slug=project_slug, slug=version_slug) except Version.DoesNotExist: ver = None if ver: # if requested version is available on main project if lang_slug != proj.language: try: translations = proj.translations.filter(language=lang_slug) if translations: ver = Version.objects.get(project__slug=translations[0].slug, slug=version_slug) else: ver = None except Version.DoesNotExist: ver = None if ver: #if requested version is available on translation project too # Case #8: Show a link to top-level page of the version suggestion['type'] = 'top' suggestion['message'] = "What are you looking for?" suggestion['href'] = proj.get_docs_url(ver.slug, lang_slug) else: # requested version is available but not in requested language # Case #7: Show available translations of the version suggestion['type'] = 'list' suggestion['message'] = "Requested page seems not to be translated in requested language. But it's available in these languages." suggestion['list'] = [] suggestion['list'].append({ 'label':proj.language, 'project': proj, 'version_slug': version_slug, 'pagename': pagename }) for t in proj.translations.all(): try: Version.objects.get(project__slug=t.slug, slug=version_slug) suggestion['list'].append({ 'label':t.language, 'project': t, 'version_slug': version_slug, 'pagename': pagename }) except Version.DoesNotExist: pass else: # requested version does not exist on main project if lang_slug == proj.language: trans = proj else: translations = proj.translations.filter(language=lang_slug) trans = translations[0] if translations else None if trans: # requested language is available # Case #6: Show available versions of the translation suggestion['type'] = 'list' suggestion['message'] = "Requested version seems not to have been built yet. But these versions are available." suggestion['list'] = [] for v in Version.objects.public(user, trans, True): suggestion['list'].append({ 'label': v.slug, 'project': trans, 'version_slug': v.slug, 'pagename': pagename }) else: # requested project exists but requested version and language are not available. # Case #5: Show a link to top-level page of default version of main project suggestion['type'] = 'top' suggestion['message'] = 'What are you looking for??' suggestion['href'] = proj.get_docs_url() except Project.DoesNotExist: # Case #1-4: Show error mssage suggestion['type'] = 'none' suggestion['message'] = "We're sorry, we don't know what you're looking for" else: suggestion['type'] = 'none' suggestion['message'] = "We're sorry, we don't know what you're looking for" return suggestion def divide_by_zero(request): return 1 / 0 def morelikethis(request, project_slug, filename): project = get_object_or_404(Project, slug=project_slug) file = get_object_or_404(ImportedFile, project=project, path=filename) # sqs = SearchQuerySet().filter(project=project).more_like_this(file)[:5] sqs = SearchQuerySet().more_like_this(file)[:5] if len(sqs): output = [(obj.title, obj.absolute_url) for obj in sqs] json_response = json.dumps(output) else: json_response = {"message": "Not Found"} jsonp = "%s(%s)" % (request.GET.get('callback'), json_response) return HttpResponse(jsonp, mimetype='text/javascript') class SearchView(TemplateView): template_name = "search/base_facet.html" results = EmptySearchQuerySet() form_class = FacetedSearchForm form = None query = '' selected_facets = None selected_facets_list = None def get_context_data(self, request, **kwargs): context = super(SearchView, self).get_context_data(**kwargs) context['request'] = self.request # causes solr request #1 context['facets'] = self.results.facet_counts() context['form'] = self.form context['query'] = self.query context['selected_facets'] = ('&'.join(self.selected_facets) if self.selected_facets else '') context['selected_facets_list'] = self.selected_facets_list context['results'] = self.results context['count'] = len(self.results) # causes solr request #2 return context def get(self, request, **kwargs): """ Performing the search causes three requests to be sent to Solr. 1. For the facets 2. For the count (unavoidable, as pagination will cause this anyay) 3. For the results """ self.request = request self.form = self.build_form() self.selected_facets = self.get_selected_facets() self.selected_facets_list = self.get_selected_facets_list() self.query = self.get_query() if self.form.is_valid(): self.results = self.get_results() context = self.get_context_data(request, **kwargs) # For returning results partials for javascript if request.is_ajax() or request.GET.get('ajax'): self.template_name = 'search/faceted_results.html' return self.render_to_response(context) def build_form(self): """ Instantiates the form the class should use to process the search query. """ data = self.request.GET if len(self.request.GET) else None return self.form_class(data, facets=('project',)) def get_selected_facets_list(self): return [tuple(s.split(':')) for s in self.selected_facets if s] def get_selected_facets(self): """ Returns the a list of facetname:value strings e.g. [u'project_exact:Read The Docs', u'author_exact:Eric Holscher'] """ return self.request.GET.getlist('selected_facets') def get_query(self): """ Returns the query provided by the user. Returns an empty string if the query is invalid. """ return self.request.GET.get('q') def get_results(self): """ Fetches the results via the form. """ return self.form.search()