| # Copyright 2010 The Go Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style |
| # license that can be found in the LICENSE file. |
| |
| # This is the server part of the package dashboard. |
| # It must be run by App Engine. |
| |
| from google.appengine.api import mail |
| from google.appengine.api import memcache |
| from google.appengine.api import taskqueue |
| from google.appengine.api import urlfetch |
| from google.appengine.api import users |
| from google.appengine.ext import db |
| from google.appengine.ext import webapp |
| from google.appengine.ext.webapp import template |
| from google.appengine.ext.webapp.util import run_wsgi_app |
| import datetime |
| import logging |
| import os |
| import re |
| import sets |
| import urllib2 |
| |
| # local imports |
| from auth import auth |
| import toutf8 |
| import const |
| |
| template.register_template_library('toutf8') |
| |
| # Storage model for package info recorded on server. |
| class Package(db.Model): |
| path = db.StringProperty() |
| web_url = db.StringProperty() # derived from path |
| count = db.IntegerProperty() # grand total |
| week_count = db.IntegerProperty() # rolling weekly count |
| day_count = db.TextProperty(default='') # daily count |
| last_install = db.DateTimeProperty() |
| |
| # data contributed by gobuilder |
| info = db.StringProperty() |
| ok = db.BooleanProperty() |
| last_ok = db.DateTimeProperty() |
| |
| def get_day_count(self): |
| counts = {} |
| if not self.day_count: |
| return counts |
| for d in str(self.day_count).split('\n'): |
| date, count = d.split(' ') |
| counts[date] = int(count) |
| return counts |
| |
| def set_day_count(self, count): |
| days = [] |
| for day, count in count.items(): |
| days.append('%s %d' % (day, count)) |
| days.sort(reverse=True) |
| days = days[:28] |
| self.day_count = '\n'.join(days) |
| |
| def inc(self): |
| count = self.get_day_count() |
| today = str(datetime.date.today()) |
| count[today] = count.get(today, 0) + 1 |
| self.set_day_count(count) |
| self.update_week_count(count) |
| self.count += 1 |
| |
| def update_week_count(self, count=None): |
| if count is None: |
| count = self.get_day_count() |
| total = 0 |
| today = datetime.date.today() |
| for i in range(7): |
| day = str(today - datetime.timedelta(days=i)) |
| if day in count: |
| total += count[day] |
| self.week_count = total |
| |
| |
| # PackageDaily kicks off the daily package maintenance cron job |
| # and serves the associated task queue. |
| class PackageDaily(webapp.RequestHandler): |
| |
| def get(self): |
| # queue a task to update each package with a week_count > 0 |
| keys = Package.all(keys_only=True).filter('week_count >', 0) |
| for key in keys: |
| taskqueue.add(url='/package/daily', params={'key': key.name()}) |
| |
| def post(self): |
| # update a single package (in a task queue) |
| def update(key): |
| p = Package.get_by_key_name(key) |
| if not p: |
| return |
| p.update_week_count() |
| p.put() |
| key = self.request.get('key') |
| if not key: |
| return |
| db.run_in_transaction(update, key) |
| |
| |
| class Project(db.Model): |
| name = db.StringProperty(indexed=True) |
| descr = db.StringProperty() |
| web_url = db.StringProperty() |
| package = db.ReferenceProperty(Package) |
| category = db.StringProperty(indexed=True) |
| tags = db.ListProperty(str) |
| approved = db.BooleanProperty(indexed=True) |
| |
| |
| re_bitbucket = re.compile(r'^(bitbucket\.org/[a-z0-9A-Z_.\-]+/[a-zA-Z0-9_.\-]+)(/[a-z0-9A-Z_.\-/]+)?$') |
| re_googlecode = re.compile(r'^[a-z0-9\-]+\.googlecode\.com/(svn|hg|git)(/[a-z0-9A-Z_.\-/]+)?$') |
| re_github = re.compile(r'^github\.com/[a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)+$') |
| re_launchpad = re.compile(r'^launchpad\.net/([a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)?|~[a-z0-9A-Z_.\-]+/(\+junk|[a-z0-9A-Z_.\-]+)/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]+)?$') |
| |
| def vc_to_web(path): |
| if re_bitbucket.match(path): |
| m = re_bitbucket.match(path) |
| check_url = 'http://' + m.group(1) + '/?cmd=heads' |
| web = 'http://' + m.group(1) + '/' |
| elif re_github.match(path): |
| m = re_github_web.match(path) |
| check_url = 'https://raw.github.com/' + m.group(1) + '/' + m.group(2) + '/master/' |
| web = 'http://github.com/' + m.group(1) + '/' + m.group(2) + '/' |
| elif re_googlecode.match(path): |
| m = re_googlecode.match(path) |
| check_url = 'http://'+path |
| if not m.group(2): # append / after bare '/hg' or '/git' |
| check_url += '/' |
| web = 'http://code.google.com/p/' + path[:path.index('.')] |
| elif re_launchpad.match(path): |
| check_url = web = 'https://'+path |
| else: |
| return False, False |
| return web, check_url |
| |
| re_bitbucket_web = re.compile(r'bitbucket\.org/([a-z0-9A-Z_.\-]+)/([a-z0-9A-Z_.\-]+)') |
| re_googlecode_web = re.compile(r'code.google.com/p/([a-z0-9\-]+)') |
| re_github_web = re.compile(r'github\.com/([a-z0-9A-Z_.\-]+)/([a-z0-9A-Z_.\-]+)') |
| re_launchpad_web = re.compile(r'launchpad\.net/([a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)?|~[a-z0-9A-Z_.\-]+/(\+junk|[a-z0-9A-Z_.\-]+)/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]+)?') |
| re_striphttp = re.compile(r'https?://(www\.)?') |
| |
| def find_googlecode_vcs(path): |
| # Perform http request to path/hg or path/git to check if they're |
| # using mercurial or git. Otherwise, assume svn. |
| for vcs in ['git', 'hg']: |
| try: |
| response = urlfetch.fetch('http://'+path+vcs, deadline=1) |
| if response.status_code == 200: |
| return vcs |
| except: pass |
| return 'svn' |
| |
| def web_to_vc(url): |
| url = re_striphttp.sub('', url) |
| m = re_bitbucket_web.match(url) |
| if m: |
| return 'bitbucket.org/'+m.group(1)+'/'+m.group(2) |
| m = re_github_web.match(url) |
| if m: |
| return 'github.com/'+m.group(1)+'/'+m.group(2) |
| m = re_googlecode_web.match(url) |
| if m: |
| path = m.group(1)+'.googlecode.com/' |
| vcs = find_googlecode_vcs(path) |
| return path + vcs |
| m = re_launchpad_web.match(url) |
| if m: |
| return m.group(0) |
| return False |
| |
| MaxPathLength = 100 |
| CacheTimeout = 3600 |
| |
| class PackagePage(webapp.RequestHandler): |
| def get(self): |
| if self.request.get('fmt') == 'json': |
| return self.json() |
| |
| html = memcache.get('view-package') |
| if not html: |
| tdata = {} |
| |
| q = Package.all().filter('week_count >', 0) |
| q.order('-week_count') |
| tdata['by_week_count'] = q.fetch(50) |
| |
| q = Package.all() |
| q.order('-last_install') |
| tdata['by_time'] = q.fetch(20) |
| |
| q = Package.all() |
| q.order('-count') |
| tdata['by_count'] = q.fetch(100) |
| |
| path = os.path.join(os.path.dirname(__file__), 'package.html') |
| html = template.render(path, tdata) |
| memcache.set('view-package', html, time=CacheTimeout) |
| |
| self.response.headers['Content-Type'] = 'text/html; charset=utf-8' |
| self.response.out.write(html) |
| |
| def json(self): |
| json = memcache.get('view-package-json') |
| if not json: |
| q = Package.all() |
| s = '{"packages": [' |
| sep = '' |
| for r in q: |
| s += '%s\n\t{"path": "%s", "last_install": "%s", "count": "%s"}' % (sep, r.path, r.last_install, r.count) |
| sep = ',' |
| s += '\n]}\n' |
| json = s |
| memcache.set('view-package-json', json, time=CacheTimeout) |
| self.response.set_status(200) |
| self.response.headers['Content-Type'] = 'text/plain; charset=utf-8' |
| self.response.out.write(json) |
| |
| def can_get_url(self, url): |
| try: |
| urllib2.urlopen(urllib2.Request(url)) |
| return True |
| except: |
| return False |
| |
| def is_valid_package_path(self, path): |
| return (re_bitbucket.match(path) or |
| re_googlecode.match(path) or |
| re_github.match(path) or |
| re_launchpad.match(path)) |
| |
| def record_pkg(self, path): |
| # sanity check string |
| if not path or len(path) > MaxPathLength or not self.is_valid_package_path(path): |
| return False |
| |
| # look in datastore |
| key = 'pkg-' + path |
| p = Package.get_by_key_name(key) |
| if p is None: |
| # not in datastore - verify URL before creating |
| web, check_url = vc_to_web(path) |
| if not web: |
| logging.error('unrecognized path: %s', path) |
| return False |
| if not self.can_get_url(check_url): |
| logging.error('cannot get %s', check_url) |
| return False |
| p = Package(key_name = key, path = path, count = 0, web_url = web) |
| |
| if auth(self.request): |
| # builder updating package metadata |
| p.info = self.request.get('info') |
| p.ok = self.request.get('ok') == "true" |
| if p.ok: |
| p.last_ok = datetime.datetime.utcnow() |
| else: |
| # goinstall reporting an install |
| p.inc() |
| p.last_install = datetime.datetime.utcnow() |
| |
| # update package object |
| p.put() |
| return True |
| |
| def post(self): |
| path = self.request.get('path') |
| ok = db.run_in_transaction(self.record_pkg, path) |
| if ok: |
| self.response.set_status(200) |
| self.response.out.write('ok') |
| else: |
| logging.error('invalid path in post: %s', path) |
| self.response.set_status(500) |
| self.response.out.write('not ok') |
| |
| class ProjectPage(webapp.RequestHandler): |
| |
| def get(self): |
| admin = users.is_current_user_admin() |
| if self.request.path == "/project/login": |
| self.redirect(users.create_login_url("/project")) |
| elif self.request.path == "/project/logout": |
| self.redirect(users.create_logout_url("/project")) |
| elif self.request.path == "/project/edit" and admin: |
| self.edit() |
| elif self.request.path == "/project/assoc" and admin: |
| self.assoc() |
| else: |
| self.list() |
| |
| def assoc(self): |
| projects = Project.all() |
| for p in projects: |
| if p.package: |
| continue |
| path = web_to_vc(p.web_url) |
| if not path: |
| continue |
| pkg = Package.get_by_key_name("pkg-"+path) |
| if not pkg: |
| self.response.out.write('no: %s %s<br>' % (p.web_url, path)) |
| continue |
| p.package = pkg |
| p.put() |
| self.response.out.write('yes: %s %s<br>' % (p.web_url, path)) |
| |
| def post(self): |
| if self.request.path == "/project/edit": |
| self.edit(True) |
| else: |
| data = dict(map(lambda x: (x, self.request.get(x)), ["name","descr","web_url"])) |
| if reduce(lambda x, y: x or not y, data.values(), False): |
| data["submitMsg"] = "You must complete all the fields." |
| self.list(data) |
| return |
| p = Project.get_by_key_name("proj-"+data["name"]) |
| if p is not None: |
| data["submitMsg"] = "A project by this name already exists." |
| self.list(data) |
| return |
| p = Project(key_name="proj-"+data["name"], **data) |
| p.put() |
| |
| path = os.path.join(os.path.dirname(__file__), 'project-notify.txt') |
| mail.send_mail( |
| sender=const.mail_from, |
| to=const.mail_submit_to, |
| subject=const.mail_submit_subject, |
| body=template.render(path, {'project': p})) |
| |
| self.list({"submitMsg": "Your project has been submitted."}) |
| |
| def list(self, additional_data={}): |
| cache_key = 'view-project-data' |
| tag = self.request.get('tag', None) |
| if tag: |
| cache_key += '-'+tag |
| data = memcache.get(cache_key) |
| admin = users.is_current_user_admin() |
| if admin or not data: |
| projects = Project.all().order('category').order('name') |
| if not admin: |
| projects = projects.filter('approved =', True) |
| projects = list(projects) |
| |
| tags = sets.Set() |
| for p in projects: |
| for t in p.tags: |
| tags.add(t) |
| |
| if tag: |
| projects = filter(lambda x: tag in x.tags, projects) |
| |
| data = {} |
| data['tag'] = tag |
| data['tags'] = tags |
| data['projects'] = projects |
| data['admin']= admin |
| if not admin: |
| memcache.set(cache_key, data, time=CacheTimeout) |
| |
| for k, v in additional_data.items(): |
| data[k] = v |
| |
| self.response.headers['Content-Type'] = 'text/html; charset=utf-8' |
| path = os.path.join(os.path.dirname(__file__), 'project.html') |
| self.response.out.write(template.render(path, data)) |
| |
| def edit(self, save=False): |
| if save: |
| name = self.request.get("orig_name") |
| else: |
| name = self.request.get("name") |
| |
| p = Project.get_by_key_name("proj-"+name) |
| if not p: |
| self.response.out.write("Couldn't find that Project.") |
| return |
| |
| if save: |
| if self.request.get("do") == "Delete": |
| p.delete() |
| else: |
| pkg_name = self.request.get("package", None) |
| if pkg_name: |
| pkg = Package.get_by_key_name("pkg-"+pkg_name) |
| if pkg: |
| p.package = pkg.key() |
| for f in ['name', 'descr', 'web_url', 'category']: |
| setattr(p, f, self.request.get(f, None)) |
| p.approved = self.request.get("approved") == "1" |
| p.tags = filter(lambda x: x, self.request.get("tags", "").split(",")) |
| p.put() |
| memcache.delete('view-project-data') |
| self.redirect('/project') |
| return |
| |
| # get all project categories and tags |
| cats, tags = sets.Set(), sets.Set() |
| for r in Project.all(): |
| cats.add(r.category) |
| for t in r.tags: |
| tags.add(t) |
| |
| self.response.headers['Content-Type'] = 'text/html; charset=utf-8' |
| path = os.path.join(os.path.dirname(__file__), 'project-edit.html') |
| self.response.out.write(template.render(path, { |
| "taglist": tags, "catlist": cats, "p": p, "tags": ",".join(p.tags) })) |
| |
| def redirect(self, url): |
| self.response.set_status(302) |
| self.response.headers.add_header("Location", url) |
| |
| def main(): |
| app = webapp.WSGIApplication([ |
| ('/package', PackagePage), |
| ('/package/daily', PackageDaily), |
| ('/project.*', ProjectPage), |
| ], debug=True) |
| run_wsgi_app(app) |
| |
| if __name__ == '__main__': |
| main() |
