Date: Mon, 12 Dec 2011 22:42:12 GMT From: Gvozdikov Veniamin <g.veniamin@googlemail.com> To: freebsd-gnats-submit@FreeBSD.org Subject: ports/163213: [new port]: databases/tarantool Tarantool, is a high performance key/value storage server Message-ID: <201112122242.pBCMgCYu062910@red.freebsd.org> Resent-Message-ID: <201112122250.pBCMoCln078076@freefall.freebsd.org>
next in thread | raw e-mail | index | archive | help
>Number: 163213 >Category: ports >Synopsis: [new port]: databases/tarantool Tarantool, is a high performance key/value storage server >Confidential: no >Severity: non-critical >Priority: low >Responsible: freebsd-ports-bugs >State: open >Quarter: >Keywords: >Date-Required: >Class: change-request >Submitter-Id: current-users >Arrival-Date: Mon Dec 12 22:50:11 UTC 2011 >Closed-Date: >Last-Modified: >Originator: Gvozdikov Veniamin >Release: FreeBSD 8.2-RELEASE >Organization: >Environment: FreeBSD 8.2-RELEASE FreeBSD 8.2-RELEASE #0 r219081M: Wed Mar 2 08:23:31 CET 2011 root@www4:/usr/obj/i386/usr/src/sys/GENERIC i386 >Description: Tarantool/Box, or simply Tarantool, is a high performance key/value storage server. The code is available for free under the terms of BSD license. Supported platforms are GNU/Linux and FreeBSD. >How-To-Repeat: >Fix: # This is a shell archive. Save it in a file, remove anything before # this line, and then unpack it by entering "sh file". Note, it may # create directories; files and directories will be owned by you and # have default permissions. # # This archive contains: # # tarantool # tarantool/distinfo # tarantool/pkg-plist # tarantool/Makefile # tarantool/pkg-descr # tarantool/files # tarantool/files/patch-test_lib_sql.py # tarantool/files/patch-test_box_tarantool.cfg # tarantool/files/patch-test_lib_tarantool_silverbox_server.py # tarantool/files/pkg-message.in # tarantool/files/tarantool.in # tarantool/files/patch-test_lib_tarantool_connection.py # tarantool/files/patch-test_lib_test_suite.py # tarantool/files/patch-test_lib_sql.g # tarantool/files/patch-test_tarantool # tarantool/files/patch-CMakeLists.txt # tarantool/files/patch-test_lib_sql_ast.py # tarantool/files/patch-test_CMakeLists.txt # echo c - tarantool mkdir -p tarantool > /dev/null 2>&1 echo x - tarantool/distinfo sed 's/^X//' >tarantool/distinfo << 'e009f9dd18370427053e903bbaeae417' XSHA256 (tarantool-1.3.5-src.tar.gz) = c78eb302eabac7b6ae04a8eadf8b2819e992d2913cdafe1a86222148982351ec XSIZE (tarantool-1.3.5-src.tar.gz) = 829327 e009f9dd18370427053e903bbaeae417 echo x - tarantool/pkg-plist sed 's/^X//' >tarantool/pkg-plist << '8c41375d2b1b0fe8ff91f493875ed017' Xbin/lib/__init__.py Xbin/lib/sql.g Xbin/lib/sql.py Xbin/lib/sql_ast.py Xbin/lib/tarantool_connection.py Xbin/lib/tarantool_preprocessor.py Xbin/lib/tarantool_silverbox_server.py Xbin/lib/test_suite.py Xbin/lib/yapps/__init__.py Xbin/lib/yapps/runtime.py Xbin/tarantool Xbin/tarantool_feeder Xbin/tarantool_silverbox X%%DOCSDIR%%/LICENSE X%%DOCSDIR%%/README X%%DOCSDIR%%/silverbox-protocol.txt X%%ETCDIR%%/tarantool.cfg X@dirrm %%DOCSDIR%% X@dirrm %%ETCDIR%% X@dirrm bin/lib/yapps X@dirrm bin/lib 8c41375d2b1b0fe8ff91f493875ed017 echo x - tarantool/Makefile sed 's/^X//' >tarantool/Makefile << 'be3ebbf6dce98e33206467475ed32ca8' X# New ports collection makefile for: tarantool X# Date created: 2011-11-25 X# Whom: Gvozdikov Veniamin <g.veniamin@googlemail.com> X# X# $FreeBSD$ X# X XPORTNAME= tarantool XPORTVERSION= 1.3.5 XCATEGORIES= databases XMASTER_SITES= http://launchpadlibrarian.net/71705094/ XDISTNAME= ${PORTNAME}-${PORTVERSION}-src X XMAINTAINER= g.veniamin@googlemail.com XCOMMENT= Tarantool, is a high performance key/value storage server X XLICENSE= BSD X XUSE_CMAKE= yes XARCH= i386 XUSE_RC_SUBR= ${PORTNAME} XSUB_FILES= pkg-message X X.include <bsd.port.pre.mk> X Xpost-patch: X @${REINPLACE_CMD} -e 's|%%DOCSDIR%%|${DOCSDIR}|g' \ X ${WRKSRC}/CMakeLists.txt X @${REINPLACE_CMD} -e 's|%%ETCDIR%%|${ETCDIR}|g' \ X ${WRKSRC}/test/CMakeLists.txt X @${RM} ${WRKSRC}/test/lib/server.py \ X ${WRKSRC}/test/lib/silverbox.py \ X ${WRKSRC}/test/lib/tarantool_admin.py \ X ${WRKSRC}/test/lib/tarantool_feeder_server.py \ X ${WRKSRC}/test/lib/tarantool_server.py \ X ${WRKSRC}/test/lib/*.orig X Xpost-install: X @${CAT} ${PKGMESSAGE} X X.include <bsd.port.post.mk> be3ebbf6dce98e33206467475ed32ca8 echo x - tarantool/pkg-descr sed 's/^X//' >tarantool/pkg-descr << '93dc114a4fc999f40d07a92d5b0fd144' XTarantool/Box, or simply Tarantool, is a high performance key/value Xstorage server. The code is available for free under the terms of XBSD license. Supported platforms are GNU/Linux and FreeBSD. X XWWW: http://tarantool.org/ 93dc114a4fc999f40d07a92d5b0fd144 echo c - tarantool/files mkdir -p tarantool/files > /dev/null 2>&1 echo x - tarantool/files/patch-test_lib_sql.py sed 's/^X//' >tarantool/files/patch-test_lib_sql.py << 'd44f42703e98a31fa4c521b8d63bf989' X--- test/lib/sql.py.orig 2011-05-14 12:16:32.000000000 +0000 X+++ test/lib/sql.py 2011-12-13 00:23:04.673107891 +0000 X@@ -30,8 +30,6 @@ X ('WHERE', re.compile('where')), X ('VALUES', re.compile('values')), X ('SET', re.compile('set')), X- ('OR', re.compile('or')), X- ('LIMIT', re.compile('limit')), X ('END', re.compile('\\s*$')), X ] X def __init__(self, str,*args,**kw): X@@ -76,16 +74,16 @@ X ident = self.ident(_context) X SET = self._scan('SET', context=_context) X update_list = self.update_list(_context) X- opt_simple_where = self.opt_simple_where(_context) X- return sql_ast.StatementUpdate(ident, update_list, opt_simple_where) X+ opt_where = self.opt_where(_context) X+ return sql_ast.StatementUpdate(ident, update_list, opt_where) X X def delete(self, _parent=None): X _context = self.Context(_parent, self._scanner, 'delete', []) X DELETE = self._scan('DELETE', context=_context) X FROM = self._scan('FROM', context=_context) X ident = self.ident(_context) X- opt_simple_where = self.opt_simple_where(_context) X- return sql_ast.StatementDelete(ident, opt_simple_where) X+ opt_where = self.opt_where(_context) X+ return sql_ast.StatementDelete(ident, opt_where) X X def select(self, _parent=None): X _context = self.Context(_parent, self._scanner, 'select', []) X@@ -94,8 +92,7 @@ X FROM = self._scan('FROM', context=_context) X ident = self.ident(_context) X opt_where = self.opt_where(_context) X- opt_limit = self.opt_limit(_context) X- return sql_ast.StatementSelect(ident, opt_where, opt_limit) X+ return sql_ast.StatementSelect(ident, opt_where) X X def ping(self, _parent=None): X _context = self.Context(_parent, self._scanner, 'ping', []) X@@ -109,8 +106,8 @@ X constant = self.constant(_context) X return (ident, constant) X X- def opt_simple_where(self, _parent=None): X- _context = self.Context(_parent, self._scanner, 'opt_simple_where', []) X+ def opt_where(self, _parent=None): X+ _context = self.Context(_parent, self._scanner, 'opt_where', []) X _token = self._peek('WHERE', 'END', context=_context) X if _token == 'END': X return None X@@ -119,38 +116,6 @@ X predicate = self.predicate(_context) X return predicate X X- def opt_where(self, _parent=None): X- _context = self.Context(_parent, self._scanner, 'opt_where', []) X- _token = self._peek('WHERE', 'LIMIT', 'END', context=_context) X- if _token != 'WHERE': X- return None X- else: # == 'WHERE' X- WHERE = self._scan('WHERE', context=_context) X- disjunction = self.disjunction(_context) X- return disjunction X- X- def disjunction(self, _parent=None): X- _context = self.Context(_parent, self._scanner, 'disjunction', []) X- predicate = self.predicate(_context) X- disjunction = [predicate] X- if self._peek('OR', 'LIMIT', 'END', context=_context) == 'OR': X- while 1: X- OR = self._scan('OR', context=_context) X- predicate = self.predicate(_context) X- disjunction.append(predicate) X- if self._peek('OR', 'LIMIT', 'END', context=_context) != 'OR': break X- return disjunction X- X- def opt_limit(self, _parent=None): X- _context = self.Context(_parent, self._scanner, 'opt_limit', []) X- _token = self._peek('LIMIT', 'END', context=_context) X- if _token == 'END': X- return 0xffffffff X- else: # == 'LIMIT' X- LIMIT = self._scan('LIMIT', context=_context) X- NUM = self._scan('NUM', context=_context) X- return int(NUM) X- X def value_list(self, _parent=None): X _context = self.Context(_parent, self._scanner, 'value_list', []) X self._scan("'\\('", context=_context) d44f42703e98a31fa4c521b8d63bf989 echo x - tarantool/files/patch-test_box_tarantool.cfg sed 's/^X//' >tarantool/files/patch-test_box_tarantool.cfg << '03d33e7b21964a28fd6683da870d15df' X--- test/box/tarantool.cfg.orig 2011-12-13 01:02:02.069760259 +0000 X+++ test/box/tarantool.cfg 2011-12-13 01:03:52.550055101 +0000 X@@ -1,11 +1,11 @@ X slab_alloc_arena = 0.1 X X-pid_file = "box.pid" X- X+pid_file = "/var/run/tarantool.pid" X+work_dir = "/var/db/tarantool" X X # Use -a not -a to work correctly on FreeBSD X # X-logger="tee -a tarantool.log" X+logger="cat >> /var/log/tarantool.log" X X primary_port = 33013 X secondary_port = 33014 03d33e7b21964a28fd6683da870d15df echo x - tarantool/files/patch-test_lib_tarantool_silverbox_server.py sed 's/^X//' >tarantool/files/patch-test_lib_tarantool_silverbox_server.py << 'e47c0a9638b1021ebba3bc5a6547bc46' X--- test/lib/tarantool_silverbox_server.py.orig 2011-05-14 12:16:32.000000000 +0000 X+++ test/lib/tarantool_silverbox_server.py 2011-12-13 00:23:04.673107891 +0000 X@@ -1,35 +1,234 @@ X+import os X+import stat X import shutil X import subprocess X-import yaml X-import ConfigParser X-from tarantool_server import TarantoolServer, TarantoolConfigFile X-from tarantool_admin import TarantoolAdmin X-from silverbox import Silverbox X- X-class TarantoolSilverboxServer(TarantoolServer): X- def __new__(cls, core="tarantool", module="silverbox"): X- return TarantoolServer.__new__(cls) X- X- def __init__(self, core="tarantool", module="silverbox"): X- TarantoolServer.__init__(self, core, module) X- X- def configure(self, config): X- TarantoolServer.configure(self, config) X- with open(self.config) as fp: X- dummy_section_name = "tarantool" X- config = ConfigParser.ConfigParser() X- config.readfp(TarantoolConfigFile(fp, dummy_section_name)) X- self.primary_port = int(config.get(dummy_section_name, "primary_port")) X- self.admin_port = int(config.get(dummy_section_name, "admin_port")) X- self.port = self.admin_port X- self.admin = TarantoolAdmin("localhost", self.admin_port) X- self.sql = Silverbox("localhost", self.primary_port) X- X- def init(self): X-# init storage X- subprocess.check_call([self.binary, "--init_storage"], X- cwd = self.vardir, X+import pexpect X+import sys X+import signal X+import time X+import socket X+import daemon X+import glob X+ X+def wait_until_connected(host, port): X+ """Wait until the server is started and accepting connections""" X+ is_connected = False X+ while not is_connected: X+ try: X+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) X+ sock.connect((host, port)) X+ is_connected = True X+ sock.close() X+ except socket.error as e: X+ time.sleep(0.001) X+ X+ X+def prepare_gdb(args): X+ """Prepare server startup arguments to run under gdb.""" X+ if "TERM" in os.environ: X+ term = os.environ["TERM"] X+ else: X+ term = "xterm" X+ X+ if term not in ["xterm", "rxvt", "urxvt", "gnome-terminal", "konsole"]: X+ raise RuntimeError("--gdb: unsupported terminal {0}".format(term)) X+ X+ args = [ term, "-e ", "gdb", "-ex", "break main", "-ex", "run"] + args X+ return args X+ X+ X+def prepare_valgrind(args, valgrind_opts): X+ "Prepare server startup arguments to run under valgrind." X+ args = ([ "valgrind", "--log-file=valgrind.log", "--quiet" ] + X+ valgrind_opts.split(" ") + args) X+ return args X+ X+ X+def check_tmpfs_exists(): X+ return os.uname()[0] in 'Linux' and os.path.isdir("/dev/shm") X+ X+def create_tmpfs_vardir(vardir): X+ os.mkdir(os.path.join("/dev/shm", vardir)) X+ os.symlink(os.path.join("/dev/shm", vardir), vardir) X+ X+class TarantoolSilverboxServer: X+ """Server represents a single server instance. Normally, the X+ program operates with only one server, but in future we may add X+ replication slaves. The server is started once at the beginning X+ of each suite, and stopped at the end.""" X+ X+ def __init__(self, args, suite_ini): X+ """Set server options: path to configuration file, pid file, exe, etc.""" X+ self.args = args X+ self.suite_ini = suite_ini X+ self.path_to_pidfile = os.path.join(args.vardir, suite_ini["pidfile"]) X+ self.path_to_exe = None X+ self.abspath_to_exe = None X+ self.is_started = False X+ X+ def install(self, silent = False): X+ """Start server instance: check if the old one exists, kill it X+ if necessary, create necessary directories and files, start X+ the server. The server working directory is taken from 'vardir', X+ specified in the prgoram options. X+ Currently this is implemented for tarantool_silverbox only.""" X+ X+ vardir = self.args.vardir X+ X+ if not silent: X+ print "Installing the server..." X+ X+ if self.path_to_exe == None: X+ self.path_to_exe = self.find_exe() X+ self.abspath_to_exe = os.path.abspath(self.path_to_exe) X+ X+ if not silent: X+ print " Found executable at " + self.path_to_exe + "." X+ X+ if not silent: X+ print " Creating and populating working directory in " +\ X+ vardir + "..." X+ X+ if os.access(vardir, os.F_OK): X+ if not silent: X+ print " Found old vardir, deleting..." X+ self.kill_old_server() X+ for filename in (glob.glob(os.path.join(vardir, "*.snap")) + X+ glob.glob(os.path.join(vardir, "*.inprogress")) + X+ glob.glob(os.path.join(vardir, "*.xlog")) + X+ glob.glob(os.path.join(vardir, "*.cfg")) + X+ glob.glob(os.path.join(vardir, "*.log")) + X+ glob.glob(os.path.join(vardir, "*.core.*")) + X+ glob.glob(os.path.join(vardir, "core"))): X+ os.remove(filename) X+ else: X+ if (self.args.mem == True and check_tmpfs_exists() and X+ os.path.basename(vardir) == vardir): X+ create_tmpfs_vardir(vardir) X+ else: X+ os.mkdir(vardir) X+ X+ shutil.copy(self.suite_ini["config"], self.args.vardir) X+ X+ subprocess.check_call([self.abspath_to_exe, "--init_storage"], X+ cwd = self.args.vardir, X # catch stdout/stderr to not clutter output X stdout = subprocess.PIPE, X stderr = subprocess.PIPE) X X+ p = subprocess.Popen([self.abspath_to_exe, "--version"], X+ cwd = self.args.vardir, X+ stdout = subprocess.PIPE) X+ version = p.stdout.read().rstrip() X+ p.wait() X+ X+ if not silent: X+ print "Starting {0} {1}.".format(os.path.basename(self.abspath_to_exe), X+ version) X+ X+ def start(self, silent = False): X+ X+ if self.is_started: X+ if not silent: X+ print "The server is already started." X+ return X+ X+ if not silent: X+ print "Starting the server..." X+ X+ args = [self.abspath_to_exe] X+ X+ if (self.args.start_and_exit and X+ not self.args.valgrind and not self.args.gdb): X+ args.append("--daemonize") X+ if self.args.gdb: X+ args = prepare_gdb(args) X+ elif self.args.valgrind: X+ args = prepare_valgrind(args, self.args.valgrind_opts) X+ X+ if self.args.start_and_exit and self.args.valgrind: X+ pid = os.fork() X+ if pid > 0: X+ os.wait() X+ else: X+ with daemon.DaemonContext(working_directory = self.args.vardir): X+ os.execvp(args[0], args) X+ else: X+ self.server = pexpect.spawn(args[0], args[1:], cwd = self.args.vardir) X+ if self.args.start_and_exit: X+ self.server.wait() X+ X+# wait until the server is connectedk X+ if self.args.gdb and self.args.start_and_exit: X+ time.sleep(1) X+ elif not self.args.start_and_exit and not self.args.gdb: X+ self.server.expect_exact("entering event loop") X+ else: X+ wait_until_connected(self.suite_ini["host"], self.suite_ini["port"]) X+ X+# Set is_started flag, to nicely support cleanup during an exception. X+ self.is_started = True X+ X+ X+ def stop(self, silent = False): X+ """Stop server instance. Do nothing if the server is not started, X+ to properly shut down the server in case of an exception during X+ start up.""" X+ if self.is_started: X+ if not silent: X+ print "Stopping the server..." X+ if self.args.gdb: X+ self.kill_old_server(True) X+ self.server.terminate() X+ self.server.expect(pexpect.EOF) X+ self.is_started = False X+ elif not silent: X+ print "The server is not started." X+ X+ def restart(self): X+ self.stop(True) X+ self.start(True) X+ X+ def test_option(self, option_list_str): X+ args = [self.abspath_to_exe] + option_list_str.split() X+ print " ".join([os.path.basename(self.abspath_to_exe)] + args[1:]) X+ output = subprocess.Popen(args, X+ cwd = self.args.vardir, X+ stdout = subprocess.PIPE, X+ stderr = subprocess.STDOUT).stdout.read() X+ print output X+ X+ X+ def find_exe(self): X+ """Locate server executable in the bindir. We just take X+ the first thing looking like an exe in there.""" X+ X+ print " Looking for server binary in {0}...".format(self.args.bindir) X+ if (os.access(self.args.bindir, os.F_OK) == False or X+ stat.S_ISDIR(os.stat(self.args.bindir).st_mode) == False): X+ raise RuntimeError("Directory " + self.args.bindir + " doesn't exist") X+ X+ for f in os.listdir(self.args.bindir): X+ f = os.path.join(self.args.bindir, f) X+ st_mode = os.stat(f).st_mode X+ if stat.S_ISREG(st_mode) and st_mode & stat.S_IXUSR: X+ return f X+ X+ raise RuntimeError("Can't find server executable in " + self.args.bindir) X+ X+ def kill_old_server(self, silent = False): X+ """Kill old server instance if it exists.""" X+ if os.access(self.path_to_pidfile, os.F_OK) == False: X+ return # Nothing to do X+ pid = 0 X+ with open(self.path_to_pidfile) as f: X+ pid = int(f.read()) X+ if not silent: X+ print " Found old server, pid {0}, killing...".format(pid) X+ try: X+ os.kill(pid, signal.SIGTERM) X+ while os.kill(pid, 0) != -1: X+ time.sleep(0.001) X+ except OSError: X+ pass X+ e47c0a9638b1021ebba3bc5a6547bc46 echo x - tarantool/files/pkg-message.in sed 's/^X//' >tarantool/files/pkg-message.in << '25ac3b3d6cbfd1d81944a0b1442f2985' X######################################################### X# X# X# After install you'll need init storage: X# X# %%PREFIX%%/bin/tarantool_silverbox --init-storage \ X# -c %%ETCDIR%%/tarantool.cfg X# X# X######################################################### 25ac3b3d6cbfd1d81944a0b1442f2985 echo x - tarantool/files/tarantool.in sed 's/^X//' >tarantool/files/tarantool.in << 'ea9a4cfad87db5d93da6ff890a26cb1f' X#!/bin/sh X# X X# PROVIDE: tarantool X# REQUIRE: DAEMON X# BEFORE: LOGIN X# KEYWORD: shutdown X# X# tarantool_enable="YES" X# tarantool_config="" X# X X. /etc/rc.subr X Xname="tarantool" X Xtarantool_enable=${tarantool_enable:-"NO"} Xtarantool_config=${tarantool_config:-"%%ETCDIR%%/$name.cfg"} X Xrcvar=`set_rcvar` X Xload_rc_config "$name" X Xcommand="%%PREFIX%%/bin/tarantool_silverbox" Xcommand_args="--daemonize --config ${tarantool_config}" Xpidfile="/var/run/$name.pid" X Xrun_rc_command "$1" X ea9a4cfad87db5d93da6ff890a26cb1f echo x - tarantool/files/patch-test_lib_tarantool_connection.py sed 's/^X//' >tarantool/files/patch-test_lib_tarantool_connection.py << 'a821f2b07cbae0352b4d42f70500715e' X--- test/lib/tarantool_connection.py.orig 1970-01-01 00:00:00.000000000 +0000 X+++ test/lib/tarantool_connection.py 2011-12-13 00:23:04.673107891 +0000 X@@ -0,0 +1,155 @@ X+__author__ = "Konstantin Osipov <kostja.osipov@gmail.com>" X+ X+# Redistribution and use in source and binary forms, with or without X+# modification, are permitted provided that the following conditions X+# are met: X+# 1. Redistributions of source code must retain the above copyright X+# notice, this list of conditions and the following disclaimer. X+# 2. Redistributions in binary form must reproduce the above copyright X+# notice, this list of conditions and the following disclaimer in the X+# documentation and/or other materials provided with the distribution. X+# X+# THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND X+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE X+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE X+# ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE X+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL X+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS X+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) X+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT X+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY X+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF X+# SUCH DAMAGE. X+ X+import socket X+import sys X+import string X+import cStringIO X+import yaml X+import re X+import sql X+import struct X+import errno X+ X+is_admin_re = re.compile("^\s*(show|save|exec|exit|reload|help)", re.I) X+ X+class AdminConnection: X+ def __init__(self, host, port): X+ self.host = host X+ self.port = port X+ self.is_connected = False X+ self.stream = cStringIO.StringIO() X+ X+ def connect(self): X+ self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) X+ self.socket.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) X+ self.socket.connect((self.host, self.port)) X+ self.is_connected = True X+ X+ def disconnect(self): X+ if self.is_connected: X+ self.socket.close() X+ self.is_connected = False X+ X+ def reconnect(self): X+ self.disconnect() X+ self.connect() X+ X+ def opt_reconnect(self): X+ """ On a socket which was disconnected, recv of 0 bytes immediately X+ returns with no data. On a socket which is alive, it returns EAGAIN. X+ Make use of this property and detect whether or not the socket is X+ dead. Reconnect a dead socket, do nothing if the socket is good.""" X+ try: X+ if self.socket.recv(0, socket.MSG_DONTWAIT) == '': X+ self.reconnect() X+ except socket.error as e: X+ if e.errno == errno.EAGAIN: X+ pass X+ else: X+ self.reconnect() X+ X+ def execute(self, command): X+ self.opt_reconnect() X+ return self.execute_no_reconnect(command) X+ X+ def execute_no_reconnect(self, command): X+ self.socket.sendall(command) X+ X+ bufsiz = 4096 X+ res = "" X+ X+ while True: X+ buf = self.socket.recv(bufsiz) X+ if not buf: X+ break X+ res = res + buf; X+ if (res.rfind("\r\n...\r\n") >= 0): X+ break X+ X+ # validate yaml by parsing it X+ yaml.load(res) X+ X+ return res X+ X+ def write(self, fragment): X+ """This is to support print >> admin, "command" syntax. X+ For every print statement, write is invoked twice: one to X+ write the command itself, and another to write \n. We should X+ accumulate all writes until we receive \n. When we receive it, X+ we execute the command, and rewind the stream.""" X+ X+ newline_pos = fragment.rfind("\n") X+ while newline_pos >= 0: X+ self.stream.write(fragment[:newline_pos+1]) X+ statement = self.stream.getvalue() X+ sys.stdout.write(statement) X+ sys.stdout.write(self.execute(statement)) X+ fragment = fragment[newline_pos+1:] X+ newline_pos = fragment.rfind("\n") X+ self.stream.seek(0) X+ self.stream.truncate() X+ X+ self.stream.write(fragment) X+ X+ def __enter__(self): X+ self.connect() X+ return self X+ X+ def __exit__(self, type, value, tb): X+ self.disconnect() X+ X+class DataConnection(AdminConnection): X+ X+ def recvall(self, length): X+ res = "" X+ while len(res) < length: X+ buf = self.socket.recv(length - len(res)) X+ res = res + buf X+ return res X+ X+ def execute_no_reconnect(self, command): X+ statement = sql.parse("sql", command) X+ if statement == None: X+ return "You have an error in your SQL syntax\n" X+ X+ payload = statement.pack() X+ header = struct.pack("<lll", statement.reqeust_type, len(payload), 0) X+ X+ self.socket.sendall(header) X+ if len(payload): X+ self.socket.sendall(payload) X+ X+ IPROTO_HEADER_SIZE = 12 X+ X+ header = self.recvall(IPROTO_HEADER_SIZE) X+ X+ response_len = struct.unpack("<lll", header)[1] X+ X+ if response_len: X+ response = self.recvall(response_len) X+ else: X+ response = None X+ X+ return statement.unpack(response) + "\n" X+ a821f2b07cbae0352b4d42f70500715e echo x - tarantool/files/patch-test_lib_test_suite.py sed 's/^X//' >tarantool/files/patch-test_lib_test_suite.py << 'e2928e6190abac07b7612ed022346e83' X--- test/lib/test_suite.py.orig 2011-05-14 12:16:32.000000000 +0000 X+++ test/lib/test_suite.py 2011-12-13 00:23:04.673107891 +0000 X@@ -10,13 +10,21 @@ X import filecmp X import shlex X import time X-from server import Server X+from tarantool_silverbox_server import TarantoolSilverboxServer X+from tarantool_connection import AdminConnection, DataConnection X import tarantool_preprocessor X import re X import cStringIO X import string X import traceback X X+class TestRunException(RuntimeError): X+ """A common exception to use across the program.""" X+ def __init__(self, message): X+ self.message = message X+ def __str__(self): X+ return self.message X+ X class FilteredStream: X """Helper class to filter .result file output""" X def __init__(self, filename): X@@ -66,12 +74,12 @@ X """Initialize test properties: path to test file, path to X temporary result file, path to the client program, test status.""" X self.name = name X- self.args = args X- self.suite_ini = suite_ini X self.result = name.replace(".test", ".result") X- self.tmp_result = os.path.join(self.args.vardir, X+ self.tmp_result = os.path.join(suite_ini["vardir"], X os.path.basename(self.result)) X self.reject = name.replace(".test", ".reject") X+ self.args = args X+ self.suite_ini = suite_ini X self.is_executed = False X self.is_executed_ok = None X self.is_equal_result = None X@@ -81,7 +89,7 @@ X """Return true if this test was run successfully.""" X return self.is_executed and self.is_executed_ok and self.is_equal_result X X- def run(self, server): X+ def run(self): X """Execute the test assuming it's a python program. X If the test aborts, print its output to stdout, and raise X an exception. Else, comprare result and reject files. X@@ -91,10 +99,18 @@ X X diagnostics = "unknown" X save_stdout = sys.stdout X+ admin = AdminConnection(self.suite_ini["host"], X+ self.suite_ini["admin_port"]) X+ sql = DataConnection(self.suite_ini["host"], X+ self.suite_ini["port"]) X+ server = self.suite_ini["server"] X try: X+ admin.connect() X+ sql.connect() X sys.stdout = FilteredStream(self.tmp_result) X- stdout_fileno = sys.stdout.stream.fileno() X- execfile(self.name, dict(locals(), **server.__dict__)) X+ server = self.suite_ini["server"] X+ vardir = self.suite_ini["vardir"] X+ execfile(self.name, globals(), locals()) X self.is_executed_ok = True X except Exception as e: X traceback.print_exc(e) X@@ -103,6 +119,8 @@ X if sys.stdout and sys.stdout != save_stdout: X sys.stdout.close() X sys.stdout = save_stdout; X+ admin.disconnect() X+ sql.disconnect() X X self.is_executed = True X X@@ -111,7 +129,7 @@ X X if self.args.valgrind: X self.is_valgrind_clean = \ X- check_valgrind_log(server.valgrind_log) == False X+ check_valgrind_log(self.suite_ini["valgrind_log"]) == False X X if self.is_executed_ok and self.is_equal_result and self.is_valgrind_clean: X print "[ pass ]" X@@ -133,12 +151,12 @@ X where = ": wrong test output" X elif not self.is_valgrind_clean: X os.remove(self.reject) X- self.print_diagnostics(server.valgrind_log, X+ self.print_diagnostics(self.suite_ini["valgrind_log"], X "Test failed! Last 10 lines of valgrind.log:") X where = ": there were warnings in valgrind.log" X X- if not self.args.is_force: X- raise RuntimeError("Failed to run test " + self.name + where) X+ if not self.suite_ini["is_force"]: X+ raise TestRunException("Failed to run test " + self.name + where) X X X def print_diagnostics(self, logfile, message): X@@ -167,6 +185,20 @@ X for line in diff: X sys.stdout.write(line) X X+class TarantoolConfigFile: X+ """ConfigParser can't read files without sections, work it around""" X+ def __init__(self, fp, section_name): X+ self.fp = fp X+ self.section_name = "[" + section_name + "]" X+ def readline(self): X+ if self.section_name: X+ section_name = self.section_name X+ self.section_name = None X+ return section_name X+ # tarantool.cfg puts string values in quote X+ return self.fp.readline().replace("\"", '') X+ X+ X class TestSuite: X """Each test suite contains a number of related tests files, X located in the same directory on disk. Each test file has X@@ -186,13 +218,15 @@ X self.args = args X self.tests = [] X self.ini = {} X- X- self.ini["core"] = "tarantool" X- self.ini["module"] = "silverbox" X+ self.ini["suite_path"] = suite_path X+ self.ini["host"] = "localhost" X+ self.ini["is_force"] = self.args.is_force X+ self.ini["vardir"] = args.vardir X+ self.ini["valgrind_log"] = os.path.join(args.vardir, "valgrind.log") X X if os.access(suite_path, os.F_OK) == False: X- raise RuntimeError("Suite \"" + suite_path +\ X- "\" doesn't exist") X+ raise TestRunException("Suite \"" + suite_path +\ X+ "\" doesn't exist") X X # read the suite config X config = ConfigParser.ConfigParser() X@@ -203,6 +237,16 @@ X self.ini["disabled"] = dict.fromkeys(self.ini["disabled"].split(" ")) X else: X self.ini["disabled"] = dict() X+# import the necessary module for test suite client X+ X+# now read the server config, we need some properties from it X+ X+ with open(self.ini["config"]) as fp: X+ dummy_section_name = "tarantool_silverbox" X+ config.readfp(TarantoolConfigFile(fp, dummy_section_name)) X+ self.ini["pidfile"] = config.get(dummy_section_name, "pid_file") X+ self.ini["admin_port"] = int(config.get(dummy_section_name, "admin_port")) X+ self.ini["port"] = int(config.get(dummy_section_name, "primary_port")) X X print "Collecting tests in \"" + suite_path + "\": " +\ X self.ini["description"] + "." X@@ -216,17 +260,9 @@ X def run_all(self): X """For each file in the test suite, run client program X assuming each file represents an individual test.""" X- try: X- server = Server(self.ini["core"], self.ini["module"]) X- except Exception as e: X- print e X- raise RuntimeError("Unknown server: core = {0}, module = {1}".format( X- self.ini["core"], self.ini["module"])) X- server.deploy(self.ini["config"], X- server.find_exe(self.args.builddir, silent=False), X- self.args.vardir, X- self.args.mem, self.args.start_and_exit, self.args.gdb, self.args.valgrind, X- silent=False) X+ server = TarantoolSilverboxServer(self.args, self.ini) X+ server.install() X+ server.start() X if self.args.start_and_exit: X print " Start and exit requested, exiting..." X exit(0) X@@ -247,7 +283,7 @@ X if os.path.basename(test.name) in self.ini["disabled"]: X print "[ skip ]" X else: X- test.run(server) X+ test.run() X if not test.passed(): X failed_tests.append(test.name) X X@@ -255,11 +291,9 @@ X if len(failed_tests): X print "Failed {0} tests: {1}.".format(len(failed_tests), X ", ".join(failed_tests)) X- server.stop(silent=False) X- server.cleanup() X+ server.stop(); X X- if self.args.valgrind and check_valgrind_log(server.valgrind_log): X+ if self.args.valgrind and check_valgrind_log(self.ini["valgrind_log"]): X print " Error! There were warnings/errors in valgrind log file:" X- print_tail_n(server.valgrind_log, 20) X- return 1 X- return len(failed_tests) X+ print_tail_n(self.ini["valgrind_log"], 20) X+ e2928e6190abac07b7612ed022346e83 echo x - tarantool/files/patch-test_lib_sql.g sed 's/^X//' >tarantool/files/patch-test_lib_sql.g << '78b4207b23345c08d3bd5dc3bfba2f7d' X--- test/lib/sql.g.orig 2011-05-14 12:16:32.000000000 +0000 X+++ test/lib/sql.g 2011-12-13 00:41:37.729004939 +0000 X@@ -5,10 +5,6 @@ X X %% X X-# The grammar below solely covers the functionality provided by X-# Tarantool binary protocol, from which follow all the X-# limitations. For reference please see doc/box-protocol.txt. X- X parser sql: X X ignore: '\\s+' X@@ -25,8 +21,6 @@ X token WHERE: 'where' X token VALUES: 'values' X token SET: 'set' X- token OR: 'or' X- token LIMIT: 'limit' X token END: '\\s*$' X X rule sql: (insert {{ stmt = insert }} | X@@ -37,27 +31,19 @@ X X rule insert: INSERT [INTO] ident VALUES value_list X {{ return sql_ast.StatementInsert(ident, value_list) }} X- rule update: UPDATE ident SET update_list opt_simple_where X- {{ return sql_ast.StatementUpdate(ident, update_list, opt_simple_where) }} X- rule delete: DELETE FROM ident opt_simple_where X- {{ return sql_ast.StatementDelete(ident, opt_simple_where) }} X- rule select: SELECT '\*' FROM ident opt_where opt_limit X- {{ return sql_ast.StatementSelect(ident, opt_where, opt_limit) }} X+ rule update: UPDATE ident SET update_list opt_where X+ {{ return sql_ast.StatementUpdate(ident, update_list, opt_where) }} X+ rule delete: DELETE FROM ident opt_where X+ {{ return sql_ast.StatementDelete(ident, opt_where) }} X+ rule select: SELECT '\*' FROM ident opt_where X+ {{ return sql_ast.StatementSelect(ident, opt_where) }} X rule ping: PING X {{ return sql_ast.StatementPing() }} X rule predicate: ident '=' constant X {{ return (ident, constant) }} X- rule opt_simple_where: {{ return None }} X+ rule opt_where: {{ return None }} X | WHERE predicate X {{ return predicate }} X- rule opt_where: {{ return None }} X- | WHERE disjunction X- {{ return disjunction }} X- rule disjunction: predicate {{ disjunction = [predicate] }} X- [(OR predicate {{ disjunction.append(predicate) }})+] X- {{ return disjunction }} X- rule opt_limit: {{ return 0xffffffff }} X- | LIMIT NUM {{ return int(NUM) }} X rule value_list: '\(' expr {{ value_list = [expr] }} X [("," expr {{ value_list.append(expr) }} )+] X '\)' {{ return value_list }} 78b4207b23345c08d3bd5dc3bfba2f7d echo x - tarantool/files/patch-test_tarantool sed 's/^X//' >tarantool/files/patch-test_tarantool << 'ef8510d3fe5d73984a6d4292baffde00' X--- test/tarantool.orig 2011-05-14 12:16:32.000000000 +0000 X+++ test/tarantool 2011-12-13 01:12:12.696699437 +0000 X@@ -1,4 +1,4 @@ X-#! /usr/bin/python X+#! /usr/bin/env python X """A simplistic client for tarantool/silverbox: administrative X console and SQL client. X X@@ -32,8 +32,8 @@ X import socket X import sys X import string X-from lib.tarantool_admin import TarantoolAdmin, is_admin_re X-from lib.box import Box X+from lib.tarantool_connection import AdminConnection, DataConnection, \ X+ is_admin_re X X class Options: X def __init__(self): X@@ -94,8 +94,8 @@ X def main(): X init_readline_history() X options = Options() X- admin_con = TarantoolAdmin(options.args.host, options.args.admin_port) X- data_con = Box(options.args.host, options.args.port) X+ admin_con = AdminConnection(options.args.host, options.args.admin_port) X+ data_con = DataConnection(options.args.host, options.args.port) X try: X admin_con.connect() X data_con.connect() ef8510d3fe5d73984a6d4292baffde00 echo x - tarantool/files/patch-CMakeLists.txt sed 's/^X//' >tarantool/files/patch-CMakeLists.txt << '1cbe30992fb1d872856097536f8c6ca2' X--- CMakeLists.txt.orig 2011-11-25 15:33:08.997444924 +0000 X+++ CMakeLists.txt 2011-11-25 15:33:30.428593855 +0000 X@@ -197,7 +197,7 @@ X add_subdirectory(test) X X install (FILES README LICENSE doc/silverbox-protocol.txt X- DESTINATION doc) X+ DESTINATION %%DOCSDIR%%) X X include (cmake/tarantool_cpack.cmake) X # 1cbe30992fb1d872856097536f8c6ca2 echo x - tarantool/files/patch-test_lib_sql_ast.py sed 's/^X//' >tarantool/files/patch-test_lib_sql_ast.py << 'b19a510736fc466d6e34980838b58ffb' X--- test/lib/sql_ast.py.orig 2011-05-14 12:16:32.000000000 +0000 X+++ test/lib/sql_ast.py 2011-12-13 00:23:04.673107891 +0000 X@@ -242,22 +242,16 @@ X class StatementSelect(StatementPing): X reqeust_type = SELECT_REQUEST_TYPE X X- def __init__(self, table_name, where, limit): X+ def __init__(self, table_name, where): X self.namespace_no = table_name X- self.index_no = None X- self.key_list = [] X- if not where: X- self.index_no = 0 X- self.key_list = ["",] X+ if where: X+ (self.index_no, key) = where X+ self.key = [key] X else: X- for (index_no, key) in where: X- self.key_list.append(key) X- if self.index_no == None: X- self.index_no = index_no X- elif self.index_no != index_no: X- raise RuntimeError("All key values in a disjunction must refer to the same index") X+ self.index_no = 0 X+ self.key = [""] X self.offset = 0 X- self.limit = limit X+ self.limit = 0xffffffff X X def pack(self): X buf = ctypes.create_string_buffer(PACKET_BUF_LEN) X@@ -266,10 +260,8 @@ X self.index_no, X self.offset, X self.limit, X- len(self.key_list)) X- offset = SELECT_REQUEST_FIXED_LEN X- for key in self.key_list: X- (buf, offset) = pack_tuple([key], buf, offset) X+ 1) X+ (buf, offset) = pack_tuple(self.key, buf, SELECT_REQUEST_FIXED_LEN) X X return buf[:offset] X b19a510736fc466d6e34980838b58ffb echo x - tarantool/files/patch-test_CMakeLists.txt sed 's/^X//' >tarantool/files/patch-test_CMakeLists.txt << 'e55c686753fa004e51acb403f7148833' X--- test/CMakeLists.txt.orig 2011-12-11 16:16:40.594230551 +0000 X+++ test/CMakeLists.txt 2011-12-11 16:19:44.915010706 +0000 X@@ -9,5 +9,4 @@ X X install (PROGRAMS tarantool DESTINATION bin) X install (DIRECTORY lib DESTINATION bin) X-install (FILES box/tarantool.cfg box/00000000000000000001.snap X- DESTINATION bin) X+install (FILES box/tarantool.cfg DESTINATION %%ETCDIR%%) e55c686753fa004e51acb403f7148833 exit >Release-Note: >Audit-Trail: >Unformatted:
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201112122242.pBCMgCYu062910>