From owner-svn-soc-all@FreeBSD.ORG Mon Aug 20 18:54:21 2012 Return-Path: Delivered-To: svn-soc-all@FreeBSD.org Received: from socsvn.FreeBSD.org (unknown [IPv6:2001:4f8:fff6::2f]) by hub.freebsd.org (Postfix) with SMTP id AD190106564A for ; Mon, 20 Aug 2012 18:54:19 +0000 (UTC) (envelope-from tzabal@FreeBSD.org) Received: by socsvn.FreeBSD.org (sSMTP sendmail emulation); Mon, 20 Aug 2012 18:54:19 +0000 Date: Mon, 20 Aug 2012 18:54:19 +0000 From: tzabal@FreeBSD.org To: svn-soc-all@FreeBSD.org MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Message-Id: <20120820185419.AD190106564A@hub.freebsd.org> Cc: Subject: socsvn commit: r240606 - in soc2012/tzabal: client-side/akcrs-head/usr.sbin/crashreport server-side/akcrs-handler server-side/akcrs-setup server-side/akcrs-website/akcrs server-side/akcrs-website/a... X-BeenThere: svn-soc-all@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: SVN commit messages for the entire Summer of Code repository List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 20 Aug 2012 18:54:21 -0000 Author: tzabal Date: Mon Aug 20 18:54:18 2012 New Revision: 240606 URL: http://svnweb.FreeBSD.org/socsvn/?view=rev&rev=240606 Log: Last commit for the GSoC period. Upload any remaining changes. Modified: soc2012/tzabal/client-side/akcrs-head/usr.sbin/crashreport/crashreport.sh soc2012/tzabal/server-side/akcrs-handler/crashreport.py soc2012/tzabal/server-side/akcrs-handler/main.py soc2012/tzabal/server-side/akcrs-handler/settings.py soc2012/tzabal/server-side/akcrs-setup/database.sql soc2012/tzabal/server-side/akcrs-setup/setup soc2012/tzabal/server-side/akcrs-website/akcrs/__init__.py soc2012/tzabal/server-side/akcrs-website/akcrs/templates/bug.mako soc2012/tzabal/server-side/akcrs-website/akcrs/templates/index.mako soc2012/tzabal/server-side/akcrs-website/akcrs/templates/login.mako soc2012/tzabal/server-side/akcrs-website/akcrs/templates/report.mako soc2012/tzabal/server-side/akcrs-website/akcrs/views.py Modified: soc2012/tzabal/client-side/akcrs-head/usr.sbin/crashreport/crashreport.sh ============================================================================== --- soc2012/tzabal/client-side/akcrs-head/usr.sbin/crashreport/crashreport.sh Mon Aug 20 17:50:20 2012 (r240605) +++ soc2012/tzabal/client-side/akcrs-head/usr.sbin/crashreport/crashreport.sh Mon Aug 20 18:54:18 2012 (r240606) @@ -335,7 +335,7 @@ if [ -f "${tmp_dir}/ddb.txt" ]; then echo "" >> ${data} echo "" >> ${data} - echo "ddb capture buffer" >> ${data} + echo "ddbcapturebuffer" >> ${data} echo "" >> ${data} echo "" >> ${data} cat "${tmp_dir}/ddb.txt" >> ${data} @@ -346,7 +346,7 @@ if [ -f "${tmp_dir}/config.txt" ]; then echo "" >> ${data} echo "" >> ${data} - echo "kernel config" >> ${data} + echo "kernelconfig" >> ${data} echo "" >> ${data} echo "" >> ${data} cat "${tmp_dir}/config.txt" >> ${data} Modified: soc2012/tzabal/server-side/akcrs-handler/crashreport.py ============================================================================== --- soc2012/tzabal/server-side/akcrs-handler/crashreport.py Mon Aug 20 17:50:20 2012 (r240605) +++ soc2012/tzabal/server-side/akcrs-handler/crashreport.py Mon Aug 20 18:54:18 2012 (r240606) @@ -88,7 +88,37 @@ self.name = None self.path = None self.info = {} - self.commands = {} + self.commands = {'crashtype': None, + 'crashdate': None, + 'hostname': None, + 'ostype': None, + 'osrelease': None, + 'version': None, + 'machine': None, + 'panic': None, + 'backtrace': None, + 'ps_axl': None, + 'vmstat_s': None, + 'vmstat_m': None, + 'vmstat_z': None, + 'vmstat_i': None, + 'pstat_T': None, + 'pstat_s': None, + 'iostat': None, + 'ipcs_a': None, + 'ipcs_T': None, + 'nfsstat': None, + 'netstat_s': None, + 'netstat_m': None, + 'netstat_id': None, + 'netstat_anr': None, + 'netstat_anA': None, + 'netstat_aL': None, + 'fstat': None, + 'dmesg': None, + 'kernelconfig': None, + 'ddbcapturebuffer': None + } def has_valid_name(self): Modified: soc2012/tzabal/server-side/akcrs-handler/main.py ============================================================================== --- soc2012/tzabal/server-side/akcrs-handler/main.py Mon Aug 20 17:50:20 2012 (r240605) +++ soc2012/tzabal/server-side/akcrs-handler/main.py Mon Aug 20 18:54:18 2012 (r240606) @@ -36,7 +36,12 @@ sender = settings.SENDER receiver = report.data.info['email'] subject = settings.SUBJECT - text = settings.TEXT % (report.id, report.confirmation_code) + + if not report.user_password: + text = settings.TEXT01 % (report.id, report.confirmation_code) + else: + text = settings.TEXT02 % (report.id, report.confirmation_code, + report.user_password) message = MIMEText(text) message['From'] = sender @@ -45,6 +50,7 @@ try: smtpconn = smtplib.SMTP(smtpserver) + print smtpserver smtpconn.sendmail(sender, receiver, message.as_string()) except smtplib.SMTPException, err: logging.info(err) @@ -81,22 +87,27 @@ elif type(report.bug_id) == type([]): report.bug_id = -1 - # Submitters - db.query = 'SELECT id FROM submitters WHERE email = %s' + # Users + db.query = 'SELECT id FROM users WHERE email = %s' db.values = (report.data.info['email'], ) if not db.execute_query(): return if db.cursor.rowcount: - report.submitter_id = db.cursor.fetchone() + report.user_id = db.cursor.fetchone() + report.user_password = None # Part of Hack else: password = generate_random_string(8) hashobj = hashlib.sha256() hashobj.update(password) hashpass = hashobj.hexdigest() - db.query = ('INSERT INTO submitters (email, password) ' + # Hack: send the password of the new submitter along with the + # confirmation email instead of sending two emails + report.user_password = password + + db.query = ('INSERT INTO users (email, password) ' 'VALUES (%s, %s)' 'RETURNING id') db.values = (report.data.info['email'], hashpass) @@ -104,13 +115,13 @@ if not db.execute_query(): return - report.submitter_id = db.cursor.fetchone() + report.user_id = db.cursor.fetchone() db.save() # Reports report.confirmation_code = generate_random_string(16) - db.query = """INSERT INTO reports (bug_id, submitter_id, confirmation_code, + db.query = """INSERT INTO reports (bug_id, user_id, confirmation_code, crashtype, crashdate, hostname, ostype, osrelease, version, machine, panic, backtrace, top_significant_func, rem_significant_funcs, ps_axl, vmstat_s, vmstat_m, vmstat_z, vmstat_i, pstat_T, pstat_s, iostat, ipcs_a, ipcs_T, @@ -121,7 +132,7 @@ RETURNING id""" db.values = (report.bug_id, - report.submitter_id, + report.user_id, report.confirmation_code, report.data.commands['crashtype'], report.data.commands['crashdate'], @@ -159,7 +170,7 @@ if not db.execute_query(): return - report.id = db.cursor.fetchone() + report.id = db.cursor.fetchone()[0] db.save() return True @@ -249,7 +260,13 @@ def recognize_report(report): - # Calculate the significant functions of the report + # The algorithm needs from every report to provide a valid panic message + # and a backtrace. If one of them is not provided, the report is marked + # as invalid. + if not report.data.commands['panic'] or not report.data.commands['backtrace']: + return + + # Calculate the significant functions of the reports significant_funcs = get_significant_funcs(report.data.commands['backtrace']) report.top_significant_func = significant_funcs[0] report.rem_significant_funcs = significant_funcs[1:] @@ -316,6 +333,7 @@ # one logged bug. If it refers to more than one bugs, then this is an # indication that our algorithm is not accurate. report.bug_id = -1 + print len(passlimit) if len(passlimit): if passlimit.count(passlimit[0]) == len(passlimit): # Refers to a known bug @@ -468,7 +486,7 @@ for filename in dirlist: path = settings.CRASHREPORTS_DIR + '/' + filename report = crashreport.CrashReport(path) - + print report if not check_report(report): move_invalid_report(report.path) continue Modified: soc2012/tzabal/server-side/akcrs-handler/settings.py ============================================================================== --- soc2012/tzabal/server-side/akcrs-handler/settings.py Mon Aug 20 17:50:20 2012 (r240605) +++ soc2012/tzabal/server-side/akcrs-handler/settings.py Mon Aug 20 18:54:18 2012 (r240606) @@ -37,8 +37,8 @@ # Confirmation email subject SUBJECT = 'Confirm your kernel crash report' -# Confirmation email text -TEXT = """\ +# Confirmation email text (for a registered user) +TEXT01 = """\ Hello, Please confirm your kernel crash report by clicking the following link: @@ -47,7 +47,24 @@ Once you confirm, your kernel crash report will be stored in our database as valid. -Thank your for your time. +Thank you for your time. +""" + +# Confirmation email text (for a new user) +TEXT02 = """\ +Hello, + +Please confirm your kernel crash report by clicking the following link: +http://akcrs.dyndns.org/confirm_report?id=%s&code=%s + +Once you confirm, your kernel crash report will be stored in our database as +valid. + +Finally, because this is the first time that you report a kernel crash to our +system, we have created for you an account with the following password: %s +Use your email address and the password to login into your account. + +Thank you for your time. """ # Panic message maximum percentage Modified: soc2012/tzabal/server-side/akcrs-setup/database.sql ============================================================================== --- soc2012/tzabal/server-side/akcrs-setup/database.sql Mon Aug 20 17:50:20 2012 (r240605) +++ soc2012/tzabal/server-side/akcrs-setup/database.sql Mon Aug 20 18:54:18 2012 (r240606) @@ -5,23 +5,28 @@ DROP TABLE reports; DROP TABLE bugs; -DROP TABLE submitters; +DROP TABLE users; +DROP TYPE bugstate; -CREATE TABLE submitters +CREATE TYPE bugstate AS ENUM ('NOSTATE', 'Open', 'Analyzed', 'Feedback', 'Closed', 'Suspended'); + + +CREATE TABLE users ( id serial NOT NULL, email varchar(254) NOT NULL, password char(64) NOT NULL, + is_developer boolean DEFAULT false, - CONSTRAINT submitters_pkey PRIMARY KEY (id) + CONSTRAINT users_pkey PRIMARY KEY (id) ); CREATE TABLE bugs ( id serial NOT NULL, - state varchar(10) NOT NULL, + state bugstate NOT NULL, reported integer NOT NULL, CONSTRAINT bugs_pkey PRIMARY KEY (id) @@ -32,8 +37,10 @@ ( id serial NOT NULL, bug_id integer NOT NULL, - submitter_id integer NOT NULL, - received_date date DEFAULT CURRENT_DATE, + user_id integer NOT NULL, + received_datetime timestamp DEFAULT CURRENT_TIMESTAMP, + --received_date date DEFAULT CURRENT_DATE, + --received_time time DEFAULT LOCALTIME, confirmation_code char(16) NOT NULL, confirmed boolean DEFAULT false, crashtype text, @@ -70,8 +77,8 @@ ddbcapturebuffer text, CONSTRAINT reports_pkey PRIMARY KEY (id), - CONSTRAINT reports_bug_id_fkey FOREIGN KEY (bug_id) REFERENCES Bugs (id), - CONSTRAINT reports_submitter_id_fkey FOREIGN KEY (submitter_id) REFERENCES Submitters (id) + CONSTRAINT reports_bug_id_fkey FOREIGN KEY (bug_id) REFERENCES bugs (id), + CONSTRAINT reports_user_id_fkey FOREIGN KEY (user_id) REFERENCES users (id) ); -INSERT INTO bugs (id, state, reported) VALUES (-1, 'Unknown', -1); \ No newline at end of file +INSERT INTO bugs (id, state, reported) VALUES (-1, 'NOSTATE', -1); \ No newline at end of file Modified: soc2012/tzabal/server-side/akcrs-setup/setup ============================================================================== --- soc2012/tzabal/server-side/akcrs-setup/setup Mon Aug 20 17:50:20 2012 (r240605) +++ soc2012/tzabal/server-side/akcrs-setup/setup Mon Aug 20 18:54:18 2012 (r240606) @@ -157,4 +157,11 @@ easy_install psycopg2 # Install the module lxml (Library for processing XML and HTML in Python) -easy_install lxml \ No newline at end of file +easy_install lxml + +# Install the pip installer +cd /usr/ports/devel/py-pip +make install clean + +# Install Django +pip install Django \ No newline at end of file Modified: soc2012/tzabal/server-side/akcrs-website/akcrs/__init__.py ============================================================================== --- soc2012/tzabal/server-side/akcrs-website/akcrs/__init__.py Mon Aug 20 17:50:20 2012 (r240605) +++ soc2012/tzabal/server-side/akcrs-website/akcrs/__init__.py Mon Aug 20 18:54:18 2012 (r240606) @@ -25,10 +25,10 @@ config.add_route('change_password', '/change_password') config.add_route('reports', '/reports') config.add_route('report', '/reports/{id}') - config.add_route('edit_report', '/report/{id}/edit_report') - config.add_route('delete_report', '/report/{id}/delete_report') + config.add_route('evaluate_report', '/reports/{id}/evaluate') config.add_route('bugs', '/bugs') config.add_route('bug', '/bugs/{id}') + config.add_route('change_bugstate', '/bugs/{id}/change_state') # Scan config.scan() return config.make_wsgi_app() \ No newline at end of file Modified: soc2012/tzabal/server-side/akcrs-website/akcrs/templates/bug.mako ============================================================================== --- soc2012/tzabal/server-side/akcrs-website/akcrs/templates/bug.mako Mon Aug 20 17:50:20 2012 (r240605) +++ soc2012/tzabal/server-side/akcrs-website/akcrs/templates/bug.mako Mon Aug 20 18:54:18 2012 (r240606) @@ -18,3 +18,19 @@ % endif

+ +% if is_developer: +

+ As a FreeBSD developer, you can change the state of the bug. Please choose below the new state.
+

+ + +
+

+% endif \ No newline at end of file Modified: soc2012/tzabal/server-side/akcrs-website/akcrs/templates/index.mako ============================================================================== --- soc2012/tzabal/server-side/akcrs-website/akcrs/templates/index.mako Mon Aug 20 17:50:20 2012 (r240605) +++ soc2012/tzabal/server-side/akcrs-website/akcrs/templates/index.mako Mon Aug 20 18:54:18 2012 (r240606) @@ -10,7 +10,8 @@
  • One bug may be referred from more than one reports

    • -

    A crash report (or problem report or ticket etc) is an indication of a problem. -Because we deal with software, and specifically kernel crashes, we call them bugs. -Our goal is to fix the bugs. The reports are the information that we have for those bugs. +

    Generally, a problem report or an issue ticket is an indication of a problem. +The ultimate goal is to fix the problem. The problem report or the issue ticket contains the information that we have for the problem. Multiple problem reports or issue tickets may refer to one problem. That means that we have more information about it. +Because our domain is kernel crashes, we use the terms kernel crash report (or report for short) and kernel bug (or bug for short). We these things in mind, have a happy browsing :)

    +

    For more information about the Automated Kernel Crash Reporting System, please visit the wiki page of the project.

    Modified: soc2012/tzabal/server-side/akcrs-website/akcrs/templates/login.mako ============================================================================== --- soc2012/tzabal/server-side/akcrs-website/akcrs/templates/login.mako Mon Aug 20 17:50:20 2012 (r240605) +++ soc2012/tzabal/server-side/akcrs-website/akcrs/templates/login.mako Mon Aug 20 18:54:18 2012 (r240606) @@ -8,7 +8,7 @@

    Invalid credentials, try again.

    % endif -

    There is no registration. In order to log in into the system, you use the email address that you have provided in the crashreport program, and the password that has been sent to you via email.

    +

    There is no registration. In order to login into the system, you use the email address that you have provided in the crashreport program, and the password that has been sent to you via email.

    Modified: soc2012/tzabal/server-side/akcrs-website/akcrs/templates/report.mako ============================================================================== --- soc2012/tzabal/server-side/akcrs-website/akcrs/templates/report.mako Mon Aug 20 17:50:20 2012 (r240605) +++ soc2012/tzabal/server-side/akcrs-website/akcrs/templates/report.mako Mon Aug 20 18:54:18 2012 (r240606) @@ -55,136 +55,144 @@ dmesg Kernel config + DDB capture buffer

    Crash type
    -
    +
    Crash date
    -
    +
    Hostname
    -
    +
    Operating System
    -
    +
    Release
    -
    +
    Version
    -
    +
    Platform
    -
    +
    Panic Message
    -
    +
    Backtrace
    -
    +
    ps -axl
    -
    +
    vmstat -s
    -
    +
    vmstat -m
    -
    +
    vmstat -z
    -
    +
    vmstat -i
    -
    +
    pstat -t
    -
    +
    pstat -s
    -
    +
    iostat
    -
    +
    ipcs -a
    -
    +
    ipcs -t
    -
    +
    nfsstat
    -
    +
    netstat -s
    -
    +
    netstat -m
    -
    +
    netstat -id
    -
    +
    netstat -anr
    -
    +
    netstat -ana
    -
    +
    netstat -al
    -
    +
    fstat
    -
    +
    dmesg
    -
    +
    Kernel config
    -
    +
    +
    +
    + DDB capture buffer
    +
    % if is_developer:

    - As a developer you can perform the following actions: -

    -Do you think that this report refers correctly to the bug with ID ${report.bug_id}? -Yes -No + As a FreeBSD developer, you can view the email address of the submitter in order to contact him for any feedback.
    + The email address of the submitter that sent this report is ${user.email}. +

    + Also, if you think that this report refers correctly to the bug with ID ${report.bug_id}, click the checkbox below.
    + Otherwise, suggest in which bug this report should point to or suggest the creation of a new bug for this report. Please write the ID of the bug below or write "new". +

    + + It refers correctly to the existing bug

    + Bug ID or "new"

    +
    +
    % endif \ No newline at end of file Modified: soc2012/tzabal/server-side/akcrs-website/akcrs/views.py ============================================================================== --- soc2012/tzabal/server-side/akcrs-website/akcrs/views.py Mon Aug 20 17:50:20 2012 (r240605) +++ soc2012/tzabal/server-side/akcrs-website/akcrs/views.py Mon Aug 20 18:54:18 2012 (r240606) @@ -116,17 +116,18 @@ id = request.matchdict['id'] query = DBSession.query(Report).filter(Report.confirmed == True, Report.id == id) report = query.one() - return {'report': report, 'logged_in': logged_in, 'is_developer': is_developer} - - -@view_config(route_name='edit_report', renderer='edit_report.mako') -def edit_report(request): - return {} + + # To have the email address of the submitter + query = DBSession.query(User).filter(User.id == report.user_id) + user = query.one() + + return {'report': report, 'user': user, 'logged_in': logged_in, 'is_developer': is_developer} -@view_config(route_name='delete_report', renderer='delete_report.mako') -def delete_report(request): - return {} +@view_config(route_name='evaluate_report', renderer='evaluate_report.mako') +def evaluate_report(request): + logged_in = authenticated_userid(request) + return {'logged_in': logged_in} @view_config(route_name='bugs', renderer='bugs.mako') @@ -139,6 +140,11 @@ @view_config(route_name='bug', renderer='bug.mako') def bug(request): logged_in = authenticated_userid(request) + is_developer = None + if logged_in: + user = DBSession.query(User).filter(User.email == logged_in).one() + is_developer = user.is_developer + id = request.matchdict['id'] # All the data related to the requested bug @@ -151,4 +157,30 @@ # The reports that refer to the requested bug query = DBSession.query(Report).filter(Report.bug_id == id, Report.confirmed == True) reports = query.all() - return {'bug': bug, 'reports': reports, 'logged_in': logged_in} + return {'bug': bug, 'reports': reports, 'logged_in': logged_in, 'is_developer': is_developer} + + +@view_config(route_name='change_bugstate', renderer='change_bugstate.mako') +def change_bugstate(request): + logged_in = authenticated_userid(request) + failed_attempt = True + id = request.matchdict['id'] + + is_developer = None + if logged_in: + user = DBSession.query(User).filter(User.email == logged_in).one() + is_developer = user.is_developer + else: + # change it to HTTPForbidden + HTTPFound(location=request.route_url('index')) + + if request.method == 'POST': + newstate = request.POST.get('bugstate') + if newstate: + query = DBSession.query(Bug).filter(Bug.reported > 0, Bug.id == id) + bug = query.one() + bug.state = newstate + DBSession.flush() + failed_attempt = False + + return {'failed_attempt': failed_attempt, 'is_developer': is_developer, 'logged_in': logged_in} \ No newline at end of file