| Current Path : /home/cgabriel/20_dev/12_procpy/dataninja/ |
Linux ift1.ift-informatik.de 5.4.0-216-generic #236-Ubuntu SMP Fri Apr 11 19:53:21 UTC 2025 x86_64 |
| Current File : //home/cgabriel/20_dev/12_procpy/dataninja/storeproc.py |
# coding: utf8
import os,re,sys,time,random,procpy
import pdb
try:
import json
except:
import pip
pip.main(["install","json"])
import json
import procpy.objpickle
VARS = {}
#**************************************************************************
class Store ():
def __init__ (self,app):
self.app = app
self.g = {}
def after (self,waittime,func,*pars):
self.app.after(waittime,func,self,*pars)
#**************************************************************************
class Storeproc (object):
def __init__ (self,app=None,dbh={},dbtable="",period=5,notifier=None):
self.dbh = dbh # data base handle
self.dbtable = dbtable
self.app = app
self.period = period
self.speedup = 1.0
self.notifier = notifier
self.act_repeats = 0
self.act_anzahl = 0
self.commit_zaehler = 0
self.max_commit = procpy.config.MAXCOMMIT_SQLITE
self.store = Store(app)
self.store.list_of_variables = {}
if not self.app: # object runs directly
pass
elif type(self.dbh) == type({}): # object store is a dictionary in memory
self.app.after(self.period,self.loop_tk)
else: # object store is a database
self.app.after(self.period,self.loop_db)
#**********************************************************
def mark (self,remark):
t = time.clock()
if 't0' in vars(self):
print(("%13.7f" % t) + " " + ("%13.9f" % (t-self.t0)) + " " + remark)
self.t0 = t
#**************************************************************************
def run (self,process_obj,parent_id=None): # start the run of the process
if 'OBJID' not in vars(process_obj) or parent_id:
process_obj.OBJID = "id" + ("%08u" % random.randint(10000001,99999900))
process_obj.__LOCKED__ = 0
process_obj.__PARENT__ = parent_id
if type(self.dbh) == type({}):
self.dbh[process_obj.OBJID] = process_obj
else:
while (0 == 0):
try:
process_obj.STARTTIME = time.time()
process_obj.DUMPTIME = time.time()
process_obj.EXCEPTION = "OK"
procpy.objpickle.insert_obj(self.dbh,self.dbtable,procpy.objpickle.to_mongo_dict(process_obj))
break
except Exception as e:
print (1111)
print (str(e))
process_obj.OBJID = "id" + ("%08u" % random.randint(10000001,99999900))
self.dbh.commit()
if not self.app:
self.loop_heap(process_obj)
return(process_obj.OBJID)
#**************************************************************************
def dbh_commit (self):
if self.commit_zaehler == 0:
self.commit_zaehler = self.max_commit
self.dbh.commit()
self.commit_zaehler = self.commit_zaehler - 1
#**************************************************************************
def anzahl (self,level=1):
if type(self.dbh) == type({}):
zaehler = 0
for o in list(self.dbh.values()):
if '__LEVEL__' in vars(o) and int(o.__LEVEL__) == int(level):
zaehler = zaehler + 1
return(zaehler)
#**************************************************************************
def property (self,objid,propname,propvalue=None):
if type(self.dbh) == type({}):
if not propvalue == None:
self.dbh[objid].__dict__[propname] = propvalue
return(self.dbh[objid].__dict__[propname])
else:
propname1 = procpy.objpickle.mask(propname)
self.dbh.execute("update " + self.dbtable + " set " + propname1 + " = ? " +
"where OBJID = ?",(propvalue,objid))
self.dbh_commit()
#**************************************************************************
def clear (self):
if type(self.dbh) == type({}):
for o in list(self.dbh.keys()):
del self.dbh[o]
else:
self.dbh.execute("delete from " + self.dbtable)
self.store.g = {}
self.dbh_commit()
#**************************************************************************
def notify (self,objid,jump=None, parentid=None):
if self.notifier:
self.notifier.notify(objid,jump,parentid)
#**************************************************************************
def mark_end (self):
if self.notifier:
self.notifier.mark_end()
#**************************************************************************
def loop_db (self):
obj = procpy.objpickle.read_obj( # Try to find and to lock an object:
self.dbh.execute("select * from " + self.dbtable + " where ____LOCKED____ = '0' and " +
str(time.time()) + " > DUMPTIME + SLEEP * " + str(self.speedup) ) )
if obj:
if self.dbh.execute(" update " + self.dbtable + " set ____LOCKED____ = 1 " +
" where OBJID = ? and ____LOCKED____ = '0'",(obj.OBJID,)).rowcount < 1:
obj = None
self.app.after(self.period,self.loop_db)
if not obj:
return()
if obj.JUMP:
jump = obj.JUMP
obj.JUMP = None
elif obj.__PARENT__: # the case of the REPEAT
# print "repeat case"
jump = ""
obj.__WAIT__ = 1
else: # no next jump, process has ended
self.notify(obj.OBJID)
jump = None
if type(jump) == type([]): # we have a fork, so we make sub-processes
self.dbh.execute(" update " + self.dbtable + " set ____LOCKED____ = " + str(len(jump)) +
" where OBJID = ? and ____LOCKED____ = '1'",(obj.OBJID,))
copyfields = "OBJCLASS"
for o in list(obj.__dict__.keys()):
if o not in ("__CHILDS__","__LOCKED__","__PARENT__","__REPEAT__","OBJID","JUMP","__WAIT__","SLEEP"):
copyfields = copyfields + "," + procpy.objpickle.mask(o)
subprocess_level = 1
if obj.__LEVEL__:
subprocess_level = int(obj.__LEVEL__) + 1
for jump1 in jump: # do the forking jumps via copy
zaehler = 0
while (0 == 0):
try:
self.dbh.execute(
" insert into " + self.dbtable +
" (OBJID,JUMP,____LOCKED____,____PARENT____,____LEVEL____,____WAIT____,STARTTIME,DURATION," + copyfields + ") " +
" select 'id" + ("%08u" % random.randint(10000001,99999900)) + "','" +
jump1 + "','0',OBJID,'" + str(subprocess_level) + "',0," + str(time.time()) + ",0.0," + copyfields +
" from " + self.dbtable + " where OBJID = ?",(obj.OBJID,) )
break
except Exception as e:
zaehler = zaehler + 1
if zaehler > 1000:
print("EVENT 11: ", str(e))
self.dbh.rollback()
return()
try: # make the commit, if no success, rollback
self.dbh_commit()
except:
self.dbh.rollback()
return()
if type(jump) in (type([]),type(None)): # after a fork or a non-existing jump nothing is to do
return()
self.notify(obj.OBJID,jump,obj.__PARENT__)
obj.DURATION = time.time() - float(obj.STARTTIME)
if jump == "": # prepare the jump for the parent repeat loop
if obj.__LEVEL__ == 1:
jump = "__init__"
else:
obj.APP = self.store
obj.self = obj
obj.SLEEP = 0
try:
obj.__class__.__dict__[jump](obj) # call the jump function
except Exception as e:
obj.EXCEPTION = str(e)
if 'TYP' in vars(obj) and type(obj.TYP) in (type(""),type("")):
try:
exec (obj.TYP+"(obj)")
except Exception as e:
obj.EXCEPTION = str(e)
# if not obj.EXCEPTION == "OK":
# obj.__WAIT__ = 1
# self.notify(obj.OBJID)
del obj.self
del obj.APP
self.dbh.execute("delete from " + self.dbtable + " where OBJID = ?",(obj.OBJID,))
obj.__LOCKED__ = 1
obj.DUMPTIME = time.time()
dump_obj = procpy.objpickle.to_mongo_dict(obj)
procpy.objpickle.insert_obj(self.dbh,self.dbtable,dump_obj)
if obj.__WAIT__ == 1:
parent_obj = procpy.objpickle.read_obj( self.dbh.execute("select * from " + self.dbtable +
" where OBJID = ?",(obj.__PARENT__,) ) )
if parent_obj and parent_obj.__LOCKED__:
try:
parent_obj.__CHILDS__[ obj.OBJID ] = obj
except:
parent_obj.__CHILDS__ = { obj.OBJID: obj }
parent_obj.__LOCKED__ = parent_obj.__LOCKED__ - 1
parent_obj.JUMP = jump
parent_obj.__WAIT__ = 2
#########################################################################################################
for o in list(obj.__dict__.keys()): # ist die frage, ob dann das child-field noch gebraucht wird ?
break
if obj.__dict__[o] and o not in ("__CHILDS__","__LOCKED__","__PARENT__","__REPEAT__","OBJID","JUMP","__WAIT__","SLEEP","EXCEPTION",
"STARTTIME", "DURATION", "PRODUCTION", "OBJCLASS", "DUMPTIME","__LEVEL__"):
try:
parent_obj.__dict__[o] = obj.__dict__[o]
except:
pass
# # inform siblings too, the problem is: changes in one sibling wont affect the other
# try:
# for child in parent
# self.dbh.execute("update " + self.dbtable + " set " + o + " =? where OBJID=?",(obj.__dict__[o],))
#########################################################################################################
if not 'EXCEPTION' in vars (parent_obj):
parent_obj.EXCEPTION = "OK"
if not obj.EXCEPTION == "OK" and parent_obj.EXCEPTION == "OK":
parent_obj.__WAIT__ = 1
parent_obj.EXCEPTION = "EXC: " + obj.EXCEPTION
if jump == "__init__": # check whether to repeat the process
if parent_obj.__REPEAT__:
parent_obj.EXCEPTION = "OK"
parent_obj.__REPEAT__ = int(parent_obj.__REPEAT__) - 1
parent_obj.__CHILDS__ = None # not necessary to store the childs from the repeat
if int(parent_obj.__REPEAT__) < 1:
parent_obj.JUMP = None
self.dbh.commit()
else:
parent_obj.JUMP = None
subprocess_level = 0
if parent_obj.__LEVEL__:
subprocess_level = parent_obj.__LEVEL__
self.act_anzahl = self.dbh.execute("select count(*) from " + self.dbtable +
" where ____LEVEL____ = ? and PRODUCTION = ?",
(subprocess_level+1,parent_obj.PRODUCTION)).fetchone()[0]
self.act_repeats = max(0,parent_obj.__REPEAT__)
for x in list(obj.__dict__.keys()):
if x in procpy.config.VIEWOBJ:
o = procpy.config.VIEWOBJ[x].AGGREGATE_FUNCTIONS
elif type(obj.__dict__[x]) in ( type(1.1), type(7) ) or x == '__WAIT__':
o = ["AVG","MIN","MAX","SUM","TREND"]
elif type(obj.__dict__[x]) in ( type('x'), type('x') ):
o = ["FIRST","LAST","SET"]
elif type(obj.__dict__[x]) == type([]):
if type(obj.__dict__[x][0]) == type([]):
o = ["XAVG","YAVG","XMIN","XMAX","YMIN","YMAX","GROSS","TREND"]
else:
o = ["AVG","MIN","MAX","GROSS","TREND"]
else:
o = []
if not x in self.store.list_of_variables:
self.store.list_of_variables[x] = []
if not o == self.store.list_of_variables[x]:
for x1 in o:
if not x1 in self.store.list_of_variables[x]:
self.store.list_of_variables[x].append(x1)
self.store.list_of_variables['OBJCLASS'] = []
# print "XX", self.store.list_of_variables
self.mark_end()
self.dbh.execute("delete from " + self.dbtable + " where OBJID = ?",(parent_obj.OBJID,))
procpy.objpickle.insert_obj(self.dbh,self.dbtable, procpy.objpickle.to_mongo_dict(parent_obj) )
else:
if 'WAIT' in vars(obj):
obj.__WAIT__ = 0
o = self.dbh.execute("update " + self.dbtable + " set ____LOCKED____ = '0' " +
"where OBJID = ? and ____LOCKED____ = '1'" ,(obj.OBJID,))
try: # make the commit, if no success, rollback
self.dbh_commit()
except:
self.dbh.rollback()
print("ERROR: DB COMMANDS NOT EXECUTED ...")
#**************************************************************************
#**************************************************************************
class XXXTk_cursor (object):
def __init__ (self):
self.zaehler = -1
#***********************************************************************
def xxfetchone (self):
result = {}
obj0 = None
while (0 == 0):
self.zaehler = self.zaehler + 1
if self.zaehler >= len(self.procruns):
return(None)
if obj0 == None:
obj0 = self.procruns[self.zaehler]
obj1 = obj0
else:
obj1 = self.procruns[self.zaehler]
for sort1 in self.qu_sort:
if not obj0[sort1] == obj1[sort1]:
break
for qu_field in self.qu_fields:
if qu_field in obj1.__dict__:
if qu_field in result:
result[qu_field].append(obj1.__dict__[qu_field])
else:
result[qu_field] = [obj1.__dict__[qu_field]]
if self.qu_sort == []:
break
for qu_field in result:
result[qu_field] = self.aggregate(self.qu_fields[qu_field].lower(),result[qu_field])
return(result)
#***********************************************************************
def xxaggregate (self,aggr_func,val_to_aggr):
erg = None
if type(val_to_aggr[0]) in [type(1),type(1.0)]:
for val in val_to_aggr:
if erg == None:
erg = val
else:
if aggr_func in ["sum","avg"]:
erg = erg + val
elif aggr_func == "max":
erg = max(erg,val)
elif aggr_func == "min":
erg = min(erg,val)
else:
erg = val
if aggr_func == "avg":
erg = float(erg)/float(len(val_to_aggr))
elif type(val_to_aggr[0]) == type(""):
if len(val_to_aggr) == 1:
erg = val_to_aggr[0]
else:
val_to_aggr.insert(0,aggr_func)
erg = " ---SPLIT--- ".join(val_to_aggr)
elif type(val_to_aggr[0]) == type([]):
erg = []
for val in val_to_aggr:
erg.append( self.aggregate(aggr_func,val) )
else:
erg = val_to_aggr[0].aggregate(aggr_func,val_to_aggr[1:])
return(erg)
#**************************************************************************
def xxloop_heap (self,process_obj):
while (0 == 0):
if '__WAIT__' in vars(process_obj):
time.sleep(process_obj.__WAIT__)
del process_obj.__WAIT__
if 'JUMP' in vars(process_obj):
jump = process_obj.JUMP
del process_obj.JUMP
if type(jump) == type([]):
jump = jump[0]
self.notify(jump)
time0 = time.time()
process_obj.__class__.__dict__[jump](process_obj)
time.sleep(max(0,0.05 - time.time() + time0))
else:
self.notify(jump)
break
#**************************************************************************
def xxquery_tk (self,qu_fields,qu_sort=[],qu_group=[],qu_filter=[]):
cursor = procpy.storeproc.Tk_cursor()
cursor.qu_fields = qu_fields
cursor.qu_sort = qu_sort
cursor.qu_group = qu_group
cursor.qu_filter = qu_filter
cursor.qu_zaehler = -1
cursor.procruns = list(self.dbh.values())
cursor.procruns.sort(cmp=lambda x,y: self.sort_tk(x,y,qu_sort))
return(cursor)
#**************************************************************************
def xxsort_tk (self,obj1,obj2,qu_sort):
for sort1 in qu_sort:
if obj1[sort1] < obj2[sort1]:
return(-1)
if obj1[sort1] > obj2[sort1]:
return(1)
return(0)
#**************************************************************************
def xxget_columns (self):
if type(self.dbh) == type({}):
return([])
else:
names = [""]
for name in self.store.list_of_variables:
if not name[0] == "_":
names.append(name)
names.sort()
return(names)
#**************************************************************************
def xxloop_tk (self):
for obj_key in self.dbh: # look for an object which can be executed
obj = self.dbh[obj_key]
if '__LOCKED__' in vars(obj) and obj.__LOCKED__ > 0:
del obj
elif 'DUMPTIME' in vars(obj) and '__WAIT__' in vars(obj) and obj.DUMPTIME + obj.__WAIT__ > time.time():
del obj
else:
obj.__LOCKED__ = 1
break
self.app.after(self.period,self.loop_tk)
if not 'obj' in vars(): # no object found for processing
return()
if 'JUMP' in vars(obj):
jump = obj.JUMP
del obj.JUMP
elif ('__PARENT__' in vars(obj) and obj.__PARENT__ ):
# print "repeat case"
jump = ""
obj.__WAIT__ = -1 # mark for repeating
else: # no next jump, process has ended
self.notify(obj.OBJID)
return()
sleep5 = None
if ('__WAIT__' in vars(obj)):
sleep5 = obj.__WAIT__
if type(jump) == type([]): # we have a fork, so we make sub-processes
if '__CHILDS__' in vars(obj): # delete former childs, so that they cannot be copied in the subprocesses
del obj.__CHILDS__
obj1 = procpy.objpickle.to_mongo_dict(obj)
obj.__LOCKED__ = len(jump)
obj.__CHILDS__ = {}
for jump1 in jump:
obj1['JUMP'] = jump1
obj1['__LEVEL__'] = 1
if '__LEVEL__' in vars(obj):
obj1['__LEVEL__'] = obj.__LEVEL__ + 1
self.run( procpy.objpickle.from_mongo_dict(obj1) ,obj.OBJID)
else:
self.notify(obj.OBJID,jump, obj.__PARENT__)
if jump == "" and obj.__LEVEL__ == 1: # prepare the jump for the parent repeat loop
jump = "__init__"
else:
obj.__class__.__dict__[jump](obj) # call the jump function
sleep1 = None
if '__WAIT__' in vars(obj):
sleep1 = obj.__WAIT__
del obj.__WAIT__
parent3 = None
if '__PARENT__' in vars(obj):
parent3 = obj.__PARENT__
if ('__PARENT__' in vars(obj) and obj.__PARENT__ and 'sleep1' in vars() and sleep1 == -1):
parent_obj = self.dbh[ obj.__PARENT__ ]
parent_obj.__LOCKED__ = parent_obj.__LOCKED__ - 1
parent_obj.__CHILDS__[ obj.OBJID ] = obj
parent_obj.JUMP = jump
parent_obj.__WAIT__ = -2
if jump == "__init__": # check whether to repeat the process
if '__REPEAT__' in vars(parent_obj):
parent_obj.__REPEAT__ = int(parent_obj.__REPEAT__) - 1
if int(parent_obj.__REPEAT__) < 1:
del parent_obj.JUMP
else:
del parent_obj.JUMP
self.act_anzahl = self.anzahl()
self.act_repeats = max(0,parent_obj.__REPEAT__)
self.mark_end()
else:
obj.__LOCKED__ = 0