Login | Register
My pages Projects Community openCollabNet

Discussions > commits > svn commit: r202 - in branches/stateengine/src: . db db/pysqlite

Project highlights: Stable Version: 1.6.1, Development Version: 1.7.6

current
Discussion topic

Back to topic list

svn commit: r202 - in branches/stateengine/src: . db db/pysqlite

Reply

Author jjneely
Full name Jack Neely
Date 2005-10-22 14:44:06 PDT
Message Author: jjneely
Date: Sat Oct 22 14:44:06 2005
New Revision: 202

Added:
   branches/stateengine/src/db/
   branches/stateengine​/src/db/__init__.py
   branches/stateengine​/src/db/pysqlite/
   branches/stateengine​/src/db/pysqlite/__i​nit__.py
   branches/stateengine​/src/db/pysqlite/pys​qlite.py
   branches/stateengine​/src/db/pysqlite/sch​ema.py
   branches/stateengine​/src/db/sedb.py
      - copied, changed from r201, branches/stateengine/src/db.py
   branches/stateengine​/src/db/specific.py
Removed:
   branches/stateengine/src/db.py
   branches/stateengine​/src/hosts.py
Modified:
   branches/stateengine​/src/config.py
   branches/stateengine​/src/hostconfig.py
   branches/stateengine​/src/se.conf
   branches/stateengine​/src/seadm.py
   branches/stateengine​/src/server.py
Log:
Refactor the database so that it looks and works like the database
setup in Current. This adds much sanity.


Modified: branches/stateengine​/src/config.py
Url: http://current.tigri​s.org/source/browse/​current/branches/sta​teengine/src/config.​py?view=diff&rev​=202&p1=branches​/stateengine/src/con​fig.py&r1=201​&p2=branches/statee​ngine/src/config.py​&r2=202
====================​====================​====================​==================
--- branches/stateengine​/src/config.py (original)
+++ branches/stateengine​/src/config.py Sat Oct 22 14:44:06 2005
@@ -18,7 +18,10 @@
 class Server(Configuration):
 
     type = "server"
-
+
+ def getDBType(self):
+ return self.get("db_type")
+
     def DBFile(self):
         return self.get("db")
 

Deleted: /branches/stateengine/src/db.py
Url: http://current.tigri​s.org/source/browse/​current/branches/sta​teengine/src/db.py?v​iew=auto&rev=201​

Added: branches/stateengine​/src/db/__init__.py
Url: http://current.tigri​s.org/source/browse/​current/branches/sta​teengine/src/db/__in​it__.py?view=auto​&rev=202
====================​====================​====================​==================
--- (empty file)
+++ branches/stateengine​/src/db/__init__.py Sat Oct 22 14:44:06 2005
@@ -0,0 +1,38 @@
+#!/usr/bin/python
+
+""" Provides a global variable and factory function to get to the backend.
+
+Copyright (c) 2002 Hunter Matthews Distributed under GPL.
+
+"""
+
+from sedb import *
+
+# This single global insures that all anyone else needs to know is
+# I hate doubled names like this, but I couldn't think of anything else
+# that was short.
+sdb = None
+
+def selectBackend(config):
+ """ Factory function to start the db type specified. """
+
+ # We could do something fancy here, like
+ # return globals()["%s_DB" % name]()
+ # but for now, we keep it simple and stupid
+
+ global sdb
+ print "Selecting database..."
+ if config.getDBType() == 'postgres':
+ import postgres
+ sdb = postgres.PostgresDB(config)
+ elif config.getDBType() == 'mysql':
+ import mysql
+ sdb = mysql.MysqlDB(config)
+ elif config.getDBType() == 'pysqlite':
+ import pysqlite
+ sdb = pysqlite.PySqliteDB(config)
+ else:
+ raise Exception("unknown backend type")
+
+## END OF LINE ##
+

Added: branches/stateengine​/src/db/pysqlite/__i​nit__.py
Url: http://current.tigri​s.org/source/browse/​current/branches/sta​teengine/src/db/pysq​lite/__init__.py?vie​w=auto&rev=202
====================​====================​====================​==================
--- (empty file)
+++ branches/stateengine​/src/db/pysqlite/__i​nit__.py Sat Oct 22 14:44:06 2005
@@ -0,0 +1,5 @@
+""" Adapter for pysqlite backend """
+
+from pysqlite import PySqliteDB
+
+## END OF LINE ##

Added: branches/stateengine​/src/db/pysqlite/pys​qlite.py
Url: http://current.tigri​s.org/source/browse/​current/branches/sta​teengine/src/db/pysq​lite/pysqlite.py?vie​w=auto&rev=202
====================​====================​====================​==================
--- (empty file)
+++ branches/stateengine​/src/db/pysqlite/pys​qlite.py Sat Oct 22 14:44:06 2005
@@ -0,0 +1,84 @@
+""" An implementation of currents backend as an sqlite database.
+
+ Sqlite is a single file 'embedded' database that supports nearly all
+ of the SQL-92 standard. Its a near perfect replacement for the old
+ shelve implementation.
+
+"""
+
+import os
+import os.path
+
+from db.specific import specificDB
+#XXX: from logger import *
+from exception import SQLite
+
+import sqlite
+import schema
+
+
+class PySqliteDB(specificDB):
+
+ def __init__(self, config):
+ """ Initialize the new database object. """
+
+ self.config = config
+ self.conn = None
+ self.cursor = None
+
+ self.db_file = self.config.DBFile()
+
+ if not os.path.exists(self.db_file):
+ self.initdb()
+ else:
+ self.getConnection()
+
+
+ def __del__(self):
+ self.disconnect()
+
+
+ def disconnect(self):
+ """ Close down the database file. """
+ if self.cursor:
+ self.cursor.close()
+ self.cursor = None
+ if self.conn:
+ self.conn.close()
+ self.conn = None
+
+
+ def initdb(self):
+ """ Do the steps necessary to create an empty database.
+
+ This should be enough to get the mod_python module working.
+ """
+ self.getConnection()
+ self.getCursor()
+
+ # initialize the database schema
+ # FIXME: Naturally, we should require some extra 'force' variable
+ # to overwrite the database here.
+ self.cursor.execute(​schema.INITDB)
+ self.conn.commit()
+
+
+ def getConnection(self):
+ if self.conn == None:
+ try:
+ self.conn = sqlite.connect(db=self.db_file)
+ except Exception, e:
+ log("Exception raised in sqlite.connect()", MANDATORY)
+ raise
+
+ return self.conn
+
+
+ def getCursor(self):
+ if self.cursor == None:
+ self.cursor = self.conn.cursor()
+
+ return self.conn.cursor()
+
+
+## END OF LINE ##

Added: branches/stateengine​/src/db/pysqlite/sch​ema.py
Url: http://current.tigri​s.org/source/browse/​current/branches/sta​teengine/src/db/pysq​lite/schema.py?view=​auto&rev=202
====================​====================​====================​==================
--- (empty file)
+++ branches/stateengine​/src/db/pysqlite/sch​ema.py Sat Oct 22 14:44:06 2005
@@ -0,0 +1,15 @@
+INITDB = """
+create table hosts (
+ hostid INTEGER PRIMARY KEY,
+ hostname char not null unique
+);
+
+create table actions (
+ actionkey INTEGER PRIMARY KEY,
+ hostid int not null,
+ queue int not null,
+ actionid text not null,
+ args text
+);
+"""
+

Copied: branches/stateengine​/src/db/sedb.py (from r201, branches/stateengine/src/db.py)
Url: http://current.tigri​s.org/source/browse/​current/branches/sta​teengine/src/db/sedb​.py?view=diff&re​v=202&p1=branche​s/stateengine/src/db​.py&r1=201&p​2=branches/stateengi​ne/src/db/sedb.py​&r2=202
====================​====================​====================​==================
--- branches/stateengine/src/db.py (original)
+++ branches/stateengine​/src/db/sedb.py Sat Oct 22 14:44:06 2005
@@ -1,100 +1,109 @@
-import os
-import os.path
-import sqlite
-
-SCHEMA = """
-create table hosts (
- hostid INTEGER PRIMARY KEY,
- hostname char not null unique
-);
-
-create table actions (
- actionkey INTEGER PRIMARY KEY,
- hostid int not null,
- queue int not null,
- actionid text not null,
- args text
-);
-"""
-
-class DB(object):
-
- def __init__(self, dbFile):
- """ Initialize the new database object. """
-
- self.dbFile = dbFile
- self.conn = None
- self.cursor = None
+import db
+import pickle
 
- if not os.path.exists(dbFile):
- self.initdb()
+class SEDB(object):
+
+ def __init__(self):
+ self.sdb = db.sdb
+ self.conn = self.sdb.getConnection()
+ self.c = self.sdb.getCursor()
+
+
+ def getHosts(self):
+ # Return an unorded lists of hostIDs
+ q = """select hostid from hosts"""
+ self.c.execute(q)
+
+ list = []
+ row = self.c.fetchone()
+ while row != None:
+ list.append(row[0])
+ row = self.c.fetchone()
+
+ return list
+
+
+ def getHost(self, id):
+ q = """select hostname from hosts where hostid=%s"""
+ self.c.execute(q, (id,))
+
+ r = self.c.fetchone()
+ if r == None:
+ return None
+ else:
+ return r[0]
+
+
+ def getHostID(self, hostname):
+ q = """select hostid from hosts where hostname=%s"""
+ self.c.execute(q, (hostname,))
+
+ r = self.c.fetchone()
+ if r == None:
+ return None
         else:
- self.getConnection()
- c = self.getCursor()
- c.close()
-
-
- def __del__(self):
- self.disconnect()
-
-
- def disconnect(self):
- """ Close down the database file. """
- if self.cursor:
- self.cursor.close()
- self.cursor = None
- if self.conn:
- self.conn.close()
- self.conn = None
-
-
- def commit(self):
- if self.conn:
- self.conn.commit()
-
-
- def initdb(self):
- """ Do the steps necessary to create an empty database.
-
- This should be enough to get the mod_python module working.
- """
- self.getConnection()
- self.getCursor()
+ return r[0]
 
- self.cursor.execute(SCHEMA)
+
+ def addHost(self, hostname):
+ q = """insert into hosts (hostname) values (%s)"""
+
+ id = self.getHostID(hostname)
+ if id != None:
+ return id
+
+ self.c.execute(q, (hostname,))
         self.conn.commit()
 
+ return self.getHostID(hostname)
 
- def getConnection(self):
- if self.conn == None:
- try:
- self.conn = sqlite.connect(db=self.dbFile)
- except Exception, e:
- print "Exception raised in sqlite.connect()"
- raise
-
- return self.conn
-
-
- def getCursor(self):
- c = self.conn.cursor()
- if self.cursor == None:
- self.cursor = self.conn.cursor()
 
- return c
-
+ def queuePut(self, hostid, action, args, kwargs):
+ q = """select queue from actions where hostid=%s
+ order by queue desc limit 1"""
+ q2 = """insert into actions (hostid, queue, actionid, args) values
+ (%s, %s, %s, %s)"""
+
+ blob = pickle.dumps((args, kwargs))
+
+ self.c.execute(q, (hostid,))
+ row = self.c.fetchone()
+ if row == None:
+ queueOrder = 0
+ else:
+ queueOrder = row[0] + 1
 
-# Test
-def main():
- file = "test.db"
+ self.c.execute(q2, (hostid, queueOrder, action, blob))
 
- if os.path.exists(file):
- os.unlink(file)
-
- d = DB(file)
- print d.getCursor()
+ self.conn.commit()
+
+
+
+ def queueGet(self, hostid):
+ q = """select actionkey, hostid, queue, actionid, args from actions
+ where hostid=%s order by queue asc limit 1"""
+
+ self.c.execute(q, (hostid,))
+ r = self.c.fetchone()
+
+ if r == None:
+ return None
+
+ ret = {}
+ args = pickle.loads(r[4])
+ ret['actionKey'] = r[0]
+ ret['hostid'] = r[1]
+ ret['actionID'] = r[3]
+ ret['args'] = args[0]
+ ret['kwargs'] = args[1]
+
+ return ret
+
+
+ def queueRemove(self, actionKey):
+ q = """ delete from actions where actionkey=%s"""
+ self.c.execute(q, (actionKey,))
+ self.conn.commit()
+
 
-if __name__ == "__main__":
- main()
 
-## END OF LINE ##

Added: branches/stateengine​/src/db/specific.py
Url: http://current.tigri​s.org/source/browse/​current/branches/sta​teengine/src/db/spec​ific.py?view=auto​&rev=202
====================​====================​====================​==================
--- (empty file)
+++ branches/stateengine​/src/db/specific.py Sat Oct 22 14:44:06 2005
@@ -0,0 +1,21 @@
+class specificDB(object):
+ """An Abstract class. For each database we whish to support that
+ module needs to provide an object that inherits from this base
+ class. This base class contains all the database specific
+ code."""
+
+ def __init__(self, config):
+ pass
+
+ def getConnection(self):
+ raise exception.CurrentDB(
+ "You cannot use the specificDB abstract class")
+
+ def getCursor(self):
+ return None
+
+ def initdb(self):
+ pass
+
+
+

Modified: branches/stateengine​/src/hostconfig.py
Url: http://current.tigri​s.org/source/browse/​current/branches/sta​teengine/src/hostcon​fig.py?view=diff​&rev=202&p1=bran​ches/stateengine/src​/hostconfig.py&r​1=201&p2=branche​s/stateengine/src/ho​stconfig.py&r2=2​02
====================​====================​====================​==================
--- branches/stateengine​/src/hostconfig.py (original)
+++ branches/stateengine​/src/hostconfig.py Sat Oct 22 14:44:06 2005
@@ -3,14 +3,12 @@
 import sha
 import config
 from pyparsing import *
-from db import DB
+from db import SEDB
 
 class HostDict(dict):
 
     def __init__(self):
         self.conf = config.Server()
- self.db = DB(self.conf.DBFile())
- self.cursor = self.db.getCursor()
         
         dict.__init__(self)
 

Deleted: /branches/stateengin​e/src/hosts.py
Url: http://current.tigri​s.org/source/browse/​current/branches/sta​teengine/src/hosts.p​y?view=auto&rev=​201

Modified: branches/stateengine​/src/se.conf
Url: http://current.tigri​s.org/source/browse/​current/branches/sta​teengine/src/se.conf​?view=diff&rev=2​02&p1=branches/s​tateengine/src/se.co​nf&r1=201&p2​=branches/stateengin​e/src/se.conf&r2​=202
====================​====================​====================​==================
--- branches/stateengine​/src/se.conf (original)
+++ branches/stateengine​/src/se.conf Sat Oct 22 14:44:06 2005
@@ -1,3 +1,4 @@
 [server]
+db_type=pysqlite
 db=se.sqlite
 configs=../configs

Modified: branches/stateengine​/src/seadm.py
Url: http://current.tigri​s.org/source/browse/​current/branches/sta​teengine/src/seadm.p​y?view=diff&rev=​202&p1=branches/​stateengine/src/sead​m.py&r1=201&​p2=branches/stateeng​ine/src/seadm.py​&r2=202
====================​====================​====================​==================
--- branches/stateengine​/src/seadm.py (original)
+++ branches/stateengine​/src/seadm.py Sat Oct 22 14:44:06 2005
@@ -2,11 +2,11 @@
 import sys
 
 try:
- import hosts
+ from db.sedb import SEDB
     from common import modules
     from common import prototypes
 except ImportError:
- import stateengine.hosts as hosts
+ import stateengine.db.sedb.SEDB as SEDB
     import stateengine.common.modules as modules
     import stateengine.common.prototypes as prototypes
 
@@ -35,11 +35,11 @@
         import server
         server.startServer()
         sys.exit()
-
- hostArray = hosts.Hosts()
+
+ db = SEDB()
 
     if o.host != "":
- id = hostArray.addHost(o.host)
+ id = db.addHost(o.host)
         sys.exit()
 
     opts.print_help()

Modified: branches/stateengine​/src/server.py
Url: http://current.tigri​s.org/source/browse/​current/branches/sta​teengine/src/server.​py?view=diff&rev​=202&p1=branches​/stateengine/src/ser​ver.py&r1=201​&p2=branches/statee​ngine/src/server.py​&r2=202
====================​====================​====================​==================
--- branches/stateengine​/src/server.py (original)
+++ branches/stateengine​/src/server.py Sat Oct 22 14:44:06 2005
@@ -8,8 +8,10 @@
 import sys
 import time
 import threading
-import hosts
 import hostconfig
+import config
+import db
+from db.sedb import SEDB
 
 # port server runs on
 port = 5741
@@ -20,9 +22,8 @@
     def __init__(self):
         print "Creating Server Object"
 
- self.hosts = hosts.Hosts()
- self.actions = hosts.ActionQueue()
         self._stop = False
+ self.db = SEDB()
         
         threading.Thread.__init__(self)
         
@@ -39,7 +40,7 @@
             # Check version from DB
             # reload if needed
 
- hostID = self.hosts.addHost(host)
+ hostID = self.db.addHost(host)
 
             stateKeys = cnf.getStateKeys()
             for state in stateKeys:
@@ -67,15 +68,15 @@
                     act, args, kwargs = q.get()
                     print "Action: %s\nArgs: %s\nKwargs: %s" % (act,
                             args, kwargs)
- self.actions.put(hostID, act, args, kwargs)
+ self.db.queuePut(hostID, act, args, kwargs)
 
                     
         
     def deployActions(self):
- for host in self.hosts:
- name = self.hosts[host]
+ for id in self.db.getHosts():
+ name = self.db.getHost(id)
             print "Deploying actions for %s" % name
- act = self.actions.get(host)
+ act = self.db.queueGet(id)
 
             if not act:
                 continue
@@ -95,7 +96,7 @@
                 continue
             
             print "Action returned: %s" % ret
- self.actions.remove(​act['actionKey'])
+ self.db.queueRemove(​act['actionKey'])
 
         
     def stop(self):
@@ -114,14 +115,14 @@
     
     def getActionTuple(self, host):
         
- id = self.hosts.getHostID(host)
+ id = self.db.getHostID(host)
         if id == None:
             return None
             
- act = self.actions.get(id)
+ act = self.db.queueGet(id)
 
         # XXX: If successful
- self.actions.remove(​act['actionKey'])
+ self.db.queueRemove(​act['actionKey'])
 
         return act['actionID'], act['args'], act['kwargs']
         
@@ -163,6 +164,9 @@
 
 def startServer():
 
+ # Init DB
+ db.selectBackend(con​fig.Server())
+
     # Start up RPC server
     rpc = RPCServer()

« Previous message in topic | 1 of 1 | Next message in topic »

Messages

Show all messages in topic

svn commit: r202 - in branches/stateengine/src: . db db/pysqlite jjneely Jack Neely 2005-10-22 14:44:06 PDT
Messages per page: