| Current Path : /home/cgabriel/20_dev/12_procpy/fibu/ |
Linux ift1.ift-informatik.de 5.4.0-216-generic #236-Ubuntu SMP Fri Apr 11 19:53:21 UTC 2025 x86_64 |
| Current File : //home/cgabriel/20_dev/12_procpy/fibu/sync_new.py |
# coding: utf8
import os,sys,re,glob,time,sqlite3,hashlib,time,base64,datetime
import procpy
import procpy.fibu
addpath = ".."
while len(addpath) < 30:
addpath
sys.path.append(addpath)
addpath = addpath + "/.."
import fibu
#******************************************************************************
class Sync (object):
def __init__ (self):
self.pythoncall = "python3"
try:
self.sortfkt = fibu.sortfkt
except:
self.sortfkt = None
try:
self.synchronized = fibu.synchronized
print("SYNCHRONIZED")
except:
self.synchronized = False
self.kontenrahmen = {}
try:
for kto in fibu.kontenrahmen.split("\n"):
m = re.search(r"^(.*?) +(.*?) *$",kto)
if m:
self.kontenrahmen[m.group(1)] = m.group(2)
except:
pass
#******************************************************************************
def generate_db (self,database_file):
print(".....>>>>>>",database_file)
self.dbh = sqlite3.connect(database_file)
self.dbh.row_factory = sqlite3.Row
cursor = self.dbh.cursor()
self.pythoncall = "python3"
cursor.execute("create table if not exists buchungen " +
"(DATUM,BETRAG,KTOA,KTOB,REMARK,ID default '')")
try:
cursor.execute("create index DATUM_IDX on buchungen (DATUM)" )
cursor.execute("create index KTOA_IDX on buchungen (KTOA)" )
cursor.execute("create index KTOB_IDX on buchungen (KTOB)" )
self.database_empty = True
except Exception as e:
self.database_empty = False
#******************************************************************************
def mark (self,remark=""):
t = time.clock()
if 't0' in vars(self):
print ( ("%9.2f" % ((t-self.t0)*1000)) + " ms for: " + remark )
self.t0 = t
#******************************************************************************
def sync3 (self,ktodirs0):
bezeichner = {}
print("")
self.mark("")
self.mark("A. Start sync")
ktodirs = []
for ktodir in ktodirs0:
if "EXCLUDE" in ktodir:
continue
ktodirs.append(re.sub(r"([^\\\/])$","\\1/",ktodir))
main_dir = self.common_string(ktodirs) # das tiefste uebergeordnete Directory ermitteln
db_dir = re.sub(r"^(.*)[\\\/].*?$","\\1",main_dir) # das erste gemeinsam erreichbare Datenbankfile ermitteln
while (0 == 0):
db_file = glob.glob(db_dir+"/*.db")
if len(db_file) == 1:
break
if len(db_file) > 1:
fehler = "ERROR: dbfile in " + db_dir + " is ambiguous"
print(fehler)
return(fehler)
if os.path.isfile(db_dir + "/fibu.py"): # oberstes Directory erreicht
db_file = None # kein Datenbankfile
break
db_dir = os.path.abspath(db_dir+"/..")
self.generated_in = None
if not db_file:
print(" ... no database found, so create it here ...")
db_file = [main_dir+"/__cache__.db"]
self.generated_in = main_dir
if main_dir in ktodirs:
ktodirs.remove(main_dir)
ktodirs.insert(0,main_dir)
# try:
# db_file = [ self.dbh_staticfile ]
# except:
# pass
self.generate_db(db_file[0])
self.mark("B. DB-File " + db_file[0])
ktolist = []
for dir in ktodirs: # jetzt die Kontoobjekte erzeugen
if not os.path.isdir(dir):
continue
print(dir)
kto = procpy.fibu.kto.Kto() # jeweils ein Konto-Objekt daraus machen
kto.find_ukto_and_interval(dir) # aus dem Pfad die Konto-ID und das Gueltigkeitsintervall herauslesen
kto.rulenr = 0 # Anzahl der Runden, die das Konto durchlaeuft
kto.rule = glob.glob(dir+"/rule.py")
kto.dir = dir
ktolist.append(kto)
if self.generated_in == main_dir:
self.generated_in = kto
self.mark("C. Generate Kto objects")
for kto in ktolist:
ktofile = glob.glob(kto.dir+"/*.kto") # jetzt den jeweiligen Kontotext einlesen
if len(ktofile) == 1: # es darf hoechstens ein kto-File im Konto-Verzeichnis geben
kto.file = kto.dir + "/" + kto.filename + ".kto"
try:
ktotext = open(ktofile[0]).read()
except:
print(ktofile[0] + " could not being opened.")
if not ktofile[0] == kto.file:
os.rename(ktofile[0],kto.file)
elif len(ktofile) == 0:
kto.file = kto.dir + "/" + kto.filename + ".kto"
if not os.path.isdir(kto.dir):
os.mkdir(kto.dir)
open(kto.file,"w").write("NEWKTO\n")
ktotext = "NEWKTO\n"
else: # im Fehlerfall mit einer Fehlermeldung Funktion beenden
fehler = "ERROR: ktofile in " + kto.dir + " is ambiguous"
print(fehler)
return(fehler)
kto.parse_ktotext(ktotext,bezeichner) # wenn das so ist, den Kontotext ins Konto-Objekt einlesen
self.mark(" > " + os.path.relpath(kto.file,"."))
self.mark("D. Parse kto files")
if self.generated_in: # das uebergeordnete Konto als primaere Quelle einlesen
self.generated_in.import_to_db(self.dbh)
self.generated_in.db0 = "xxxxxx"
ktolist_rest = []
ktolist_import = []
for kto in ktolist:
kto.id_new = kto.kto_id(kto.dir)
m = re.search(r"^[^\n]+\(([A-Za-z0-9\_\+\-]{6})([A-Za-z0-9\_\+\-]{6})\)",kto.ktotext)
if m:
kto.id = m.group(1) # der abgespeicherte Kto-Hash
kto.db = m.group(2) # der Kto-Hash des Parents
if kto.id == kto.id_new:
ktolist_import.insert(0,kto)
else:
ktolist_import.append(kto)
if kto == self.generated_in:
kto.db0 = kto.db
continue
if kto.ktotext[0:6] == "NEWKTO":
kto.id = kto.id_new
kto.db = "yyyyyy"
continue
ktolist_rest.append(kto)
self.mark("D1. Make database entries")
for kto in ktolist_import:
kto.delete_from_db(self.dbh)
kto.import_to_db(self.dbh) # Kontoinhalt in Datenbank eintragen
for kto in ktolist_rest:
kto.id = "xxxxxx"
buchungen = kto.export_from_db(self.dbh) # Kontoinhalt aus der Datenbank auslesen
kto.buchungen = buchungen
kto.db = kto.db_id(buchungen)
# print("A",m,kto.ukto,kto.id,kto.db,"---",kto.id_new)
self.mark("E. Evaluate kto hashes")
fehlerliste = []
rounds = 0
while (0 == 0):
rounds = rounds + 1
print("\n\n*****************************\nROUND: " + ("%2u" % rounds) + "\n---------\n\n")
all_synced = True
for kto in ktolist:
if kto in fehlerliste:
continue
# print(kto.ukto,kto.rule)
buchungen = kto.export_from_db(self.dbh)
# print(buchungen)
kto.db_new = kto.db_id(buchungen)
if not kto.db_new == kto.db and not kto.id_new == kto.id:
kto.db_new = kto.db
print(kto.ukto,"---",kto.id,kto.db,"---",kto.id_new,kto.db_new)
if kto.db_new == kto.db and kto.id_new == kto.id:
continue
elif not kto.db_new == kto.db and kto.id_new == kto.id: # or self.synchronized:
print(" ... write " + kto.ukto)
kto.buchungen = buchungen
saldo = kto.write_ktotext(bezeichner,self.sortfkt) # Kontotext erzeugen
# print(kto.ukto,"KTOTEXT:",kto.ktotext)
kto.db = kto.db_new
kto.id = kto.kto_id(kto.dir)
open(kto.file,"w").write(
("%-40s" % kto.filename) + "(" + kto.id + kto.db_new + ")" +
# " Saldo: " +
("%15.2f" % saldo) +
"\n\n" + kto.ktotext )
if os.path.isfile(kto.filename+".csv"):
open(kto.filename+".csv","w").write(kto.write_csv(kto.filename))
if len(kto.rule) == 1:
m = re.search(r"^(.*)[\\\/](.*)$","/"+kto.rule[0])
if m:
rule1 = m.group(2)
print(" ... run rule " + str(kto.rulenr) +" for " + kto.ukto)
os.system("chdir " + kto.dir + "; cd " + kto.dir + "; " +
self.pythoncall + " " + rule1 + " " + str(kto.rulenr) )
kto.rulenr = kto.rulenr + 1
kto.parse_ktotext( open(kto.file).read(), bezeichner )
kto.id_new = kto.kto_id(kto.dir)
all_synced = False
if not kto.id_new == kto.id: # gleich kto neu schreiben
kto.delete_from_db(self.dbh)
kto.import_to_db(self.dbh)
kto.db = "yyyyyy" #
kto.id = kto.id_new #
elif kto.db_new == kto.db and not kto.id_new == kto.id or self.synchronized:
print(" ... update " + kto.ukto)
kto.delete_from_db(self.dbh)
kto.parse_ktotext( open(kto.file).read(), bezeichner )
kto.import_to_db(self.dbh)
self.dbh.commit()
kto.db = "yyyyyy"
kto.id = kto.id_new
all_synced = False
elif not kto.db_new == kto.db and not kto.id_new == kto.id:
fehler = "ERROR: Conflict in " + kto.dir
print(fehler)
fehlerliste.append(kto)
self.dbh.commit()
self.mark("F"+str(rounds)+". Round " + str(rounds) + " finished.")
if all_synced:
print(" ... all accounts synced")
break
self.dbh.commit()
self.mark("G. All accounts synced. DB Commit")
print("")
if False and self.generated_in:
text = open(self.generated_in.file).read().split("\n")
m = re.search(r"^(.*\()([A-Za-z0-9\_\+\-]{6})([A-Za-z0-9\_\+\-]{6})(\).*)$",text[0])
if m:
# text[0] = m.group(1) + m.group(2) + self.generated_in.db0 + m.group(4)
text[0] = m.group(1) + "xxxxxx" + m.group(3) + m.group(4)
open(self.generated_in.file,"w").write("\n".join(text))
return(fehlerliste)
#******************************************************************************
def sync4 (self,ktodirs0):
bezeichner = {}
print("")
self.mark("")
self.mark("A. Start sync")
ktodirs = []
for ktodir in ktodirs0:
if "EXCLUDE" in ktodir:
continue
ktodirs.append(re.sub(r"([^\\\/])$","\\1/",ktodir))
self.generate_db(":memory:")
self.mark("B. DB-File " + db_file[0])
ktolist = []
for dir in ktodirs: # jetzt die Kontoobjekte erzeugen
if not os.path.isdir(dir):
continue
print(dir)
kto = procpy.fibu.kto.Kto() # jeweils ein Konto-Objekt daraus machen
kto.find_ukto_and_interval(dir) # aus dem Pfad die Konto-ID und das Gueltigkeitsintervall herauslesen
kto.rulenr = 0 # Anzahl der Runden, die das Konto durchlaeuft
kto.rule = glob.glob(dir+"/rule.py")
kto.dir = dir
ktolist.append(kto)
self.mark("C. Generate Kto objects")
for kto in ktolist:
ktofile = glob.glob(kto.dir+"/*.kto") # jetzt den jeweiligen Kontotext einlesen
if len(ktofile) == 1: # es darf hoechstens ein kto-File im Konto-Verzeichnis geben
kto.file = kto.dir + "/" + kto.filename + ".kto"
try:
ktotext = open(ktofile[0]).read()
except:
print(ktofile[0] + " could not being opened.")
if not ktofile[0] == kto.file:
os.rename(ktofile[0],kto.file)
elif len(ktofile) == 0:
kto.file = kto.dir + "/" + kto.filename + ".kto"
if not os.path.isdir(kto.dir):
os.mkdir(kto.dir)
open(kto.file,"w").write("NEWKTO\n")
ktotext = "NEWKTO\n"
else: # im Fehlerfall mit einer Fehlermeldung Funktion beenden
fehler = "ERROR: ktofile in " + kto.dir + " is ambiguous"
print(fehler)
return(fehler)
kto.parse_ktotext(ktotext,bezeichner) # wenn das so ist, den Kontotext ins Konto-Objekt einlesen
self.mark(" > " + os.path.relpath(kto.file,"."))
self.mark("D. Parse kto files")
for kto in ktolist:
kto.id_new = kto.kto_id(kto.dir)
m = re.search(r"^[^\n]+\(([A-Za-z0-9\_\+\-]{6})([A-Za-z0-9\_\+\-]{6})\)",kto.ktotext)
if m:
kto.id = m.group(1) # der abgespeicherte Kto-Hash
kto.db = m.group(2) # der Kto-Hash des Parents
else:
kto.db = "yyyyyy"
if kto.ktotext[0:6] == "NEWKTO":
kto.id = kto.id_new
else:
kto.id = "xxxxxx"
@@@@
self.mark("E. Evaluate kto hashes")
fehlerliste = []
rounds = 0
while (0 == 0):
rounds = rounds + 1
print("\n\n*****************************\nROUND: " + ("%2u" % rounds) + "\n---------\n\n")
all_synced = True
for kto in ktolist:
if kto in fehlerliste:
continue
# print(kto.ukto,kto.rule)
buchungen = kto.export_from_db(self.dbh)
# print(buchungen)
kto.db_new = kto.db_id(buchungen)
if not kto.db_new == kto.db and not kto.id_new == kto.id:
kto.db_new = kto.db
print(kto.ukto,"---",kto.id,kto.db,"---",kto.id_new,kto.db_new)
if kto.db_new == kto.db and kto.id_new == kto.id:
continue
elif not kto.db_new == kto.db and kto.id_new == kto.id: # or self.synchronized:
print(" ... write " + kto.ukto)
kto.buchungen = buchungen
saldo = kto.write_ktotext(bezeichner,self.sortfkt) # Kontotext erzeugen
# print(kto.ukto,"KTOTEXT:",kto.ktotext)
kto.db = kto.db_new
kto.id = kto.kto_id(kto.dir)
open(kto.file,"w").write(
("%-40s" % kto.filename) + "(" + kto.id + kto.db_new + ")" +
# " Saldo: " +
("%15.2f" % saldo) +
"\n\n" + kto.ktotext )
if os.path.isfile(kto.filename+".csv"):
open(kto.filename+".csv","w").write(kto.write_csv(kto.filename))
if len(kto.rule) == 1:
m = re.search(r"^(.*)[\\\/](.*)$","/"+kto.rule[0])
if m:
rule1 = m.group(2)
print(" ... run rule " + str(kto.rulenr) +" for " + kto.ukto)
os.system("chdir " + kto.dir + "; cd " + kto.dir + "; " +
self.pythoncall + " " + rule1 + " " + str(kto.rulenr) )
kto.rulenr = kto.rulenr + 1
kto.parse_ktotext( open(kto.file).read(), bezeichner )
kto.id_new = kto.kto_id(kto.dir)
all_synced = False
if not kto.id_new == kto.id: # gleich kto neu schreiben
kto.delete_from_db(self.dbh)
kto.import_to_db(self.dbh)
kto.db = "yyyyyy" #
kto.id = kto.id_new #
elif kto.db_new == kto.db and not kto.id_new == kto.id or self.synchronized:
print(" ... update " + kto.ukto)
kto.delete_from_db(self.dbh)
kto.parse_ktotext( open(kto.file).read(), bezeichner )
kto.import_to_db(self.dbh)
self.dbh.commit()
kto.db = "yyyyyy"
kto.id = kto.id_new
all_synced = False
elif not kto.db_new == kto.db and not kto.id_new == kto.id:
fehler = "ERROR: Conflict in " + kto.dir
print(fehler)
fehlerliste.append(kto)
self.dbh.commit()
self.mark("F"+str(rounds)+". Round " + str(rounds) + " finished.")
if all_synced:
print(" ... all accounts synced")
break
self.dbh.commit()
self.mark("G. All accounts synced. DB Commit")
print("")
if False and self.generated_in:
text = open(self.generated_in.file).read().split("\n")
m = re.search(r"^(.*\()([A-Za-z0-9\_\+\-]{6})([A-Za-z0-9\_\+\-]{6})(\).*)$",text[0])
if m:
# text[0] = m.group(1) + m.group(2) + self.generated_in.db0 + m.group(4)
text[0] = m.group(1) + "xxxxxx" + m.group(3) + m.group(4)
open(self.generated_in.file,"w").write("\n".join(text))
return(fehlerliste)
#******************************************************************************
def common_string (self,elements):
main_el = elements[0]
for el in elements: # jetzt das tiefste uebergeordnete Directory ermitteln
main_el = main_el[0:len(el)]
el1 = el[0:len(main_el)]
while (0 == 0):
if el1 == main_el:
break
el1 = el1[:-1]
main_el = main_el[:-1]
return(main_el)
#******************************************************************************
def sort (self,pattern,file):
if pattern == "..":
self.sort_numbers(file)
return()
text = open(file).read()
text_match = []
text_rest = []
for zeile in text.split("\n"):
if re.search(pattern,zeile,re.IGNORECASE):
text_match.append(zeile)
else:
text_rest.append(zeile)
text = "\n".join(text_rest) + "\n" + "\n".join(text_match) + "\n"
if os.path.isfile(file+"~"):
os.unlink(file+"~")
# os.rename(file,file+"~")
open(file,"w").write(text)
#******************************************************************************
def sort_numbers (self,file):
text = open(file).read()
text_match = []
text_rest = []
for zeile in text.split("\n"):
m = re.search(r"\d\d\d\d\d\d\d\d +\-?(\d+\.\d\d) ",zeile)
if m:
text_match.append([zeile,float(m.group(1))])
else:
text_rest.append(zeile)
text_match.sort(key=lambda x:x[1])
text_match1 = []
for tt in text_match:
text_match1.append(tt[0])
text = "\n".join(text_rest) + "\n" + "\n".join(text_match1) + "\n"
if os.path.isfile(file+"~"):
os.unlink(file+"~")
# os.rename(file,file+"~")
open(file,"w").write(text)
#*************************************************************************************
def find_quittung (self,daydiff,file,pattern=""):
buchungen = []
text = []
text1 = []
ktotext = open(file).read()
for zeile in ktotext.split("\n"):
m = re.search(r"^(\d\d\d\d\d\d\d\d) +(\-?\d+\.\d\d) +(\S+) +(\S+) +(\-?\d+\.\d\d) +(.*)",zeile)
if not m:
text.append(zeile)
continue
else:
datum = m.group(1)
betrag = m.group(2)
ktoa = m.group(3)
ktob = m.group(4)
remark = m.group(6)
buchungen.append([0,datum,betrag,ktoa,ktob,remark,zeile])
day0 = datetime.datetime.strptime("19700101","%Y%m%d")
try:
day = datetime.datetime.strptime(datum,"%Y%m%d")
except:
day = datetime.datetime.strptime(datum[0:6]+"28","%Y%m%d")
buchungen[-1][0] = (day-day0).days
while (0 == 0):
if buchungen[-1][0] - buchungen[0][0] < int(daydiff):
break
text.append(buchungen.pop(0)[-1])
'''
if abs(float(buchung[2])) == abs(float(buchungen[-1][2])):
buchung1 = buchung
buchung2 = buchungen[-1]
common_kto1 = ""
common_kto2 = ""
m1 = re.search("("+pattern+")",buchung1[-1])
if m1:
common_kto1 = m1.group(1)
m2 = re.search("("+pattern+")",buchung2[-1])
if m2:
common_kto2 = m2.group(1)
print(common_kto1,common_kto2)
print(buchung1[-1])
print(buchung2[-1])
if not common_kto1 == common_kto2: # matching!
text1.append("")
if pattern in buchung2[-1]:
buchung0 = buchung1
buchung1 = buchung2
buchung2 = buchung0
betrag1 = buchung1[2]
ktoa1 = buchung1[3]
ktob1 = buchung1[4]
if not betrag1[0] == "-":
betrag1 = "-" + betrag1
ktox = ktoa1
ktoa1 = ktob1
ktob1 = ktox
betrag2 = buchung1[2]
ktoa2 = buchung1[3]
ktob2 = buchung1[4]
if not betrag2[0] == "-":
betrag2 = "-" + betrag2
ktox = ktoa2
ktoa2 = ktob2
ktob2 = ktox
if ktob1 == standard_kto:
ktob1 = ktob2
if ktob2 == standard_kto:
ktob2 = ktob1
ktob2 = ktoa1
zeile1 = buchung1[1] + " " + betrag1 + " " + ktoa1 + " " + ktob1 + " 0.00 " + buchung1[5]
zeile2 = buchung2[1] + " " + betrag2 + " " + ktoa2 + " " + ktob2 + " 0.00 " + buchung2[5]
text1.append(zeile1)
text1.append(zeile2)
'''
standard_kto = "-13-9999"
for buchung in buchungen[:-1]:
if abs(float(buchung[2])) == abs(float(buchungen[-1][2])):
buchung1 = buchung
buchung2 = buchungen[-1]
common_kto1 = ""
common_kto2 = ""
m1 = re.search("("+pattern+")",buchung1[-1])
if m1:
common_kto1 = m.group(1)
m2 = re.search("("+pattern+")",buchung2[-1])
if m2:
common_kto2 = m.group(1)
if not common_kto1 == common_kto2:
text1.append("")
if pattern in buchung2[-1]:
buchung0 = buchung1
buchung1 = buchung2
buchung2 = buchung0
betrag1 = buchung1[2]
ktoa1 = buchung1[3]
ktob1 = buchung1[4]
if not betrag1[0] == "-":
betrag1 = "-" + betrag1
ktox = ktoa1
ktoa1 = ktob1
ktob1 = ktox
betrag2 = buchung2[2]
ktoa2 = buchung2[3]
ktob2 = buchung2[4]
if not betrag2[0] == "-":
betrag2 = "-" + betrag2
ktox = ktoa2
ktoa2 = ktob2
ktob2 = ktox
if ktob1 == standard_kto:
ktob1 = ktob2
if ktob2 == standard_kto:
ktob2 = ktob1
ktob2 = re.sub("QUIT","ZAHL",ktoa1)
zeile1 = buchung1[1] + " " + betrag1 + " " + ktoa1 + " " + ktob1 + " 0.00 " + buchung1[5]
zeile2 = buchung2[1] + " " + betrag2 + " " + ktoa2 + " " + ktob2 + " 0.00 " + buchung2[5]
if ktoa1 == ktob1 or ktoa2 == ktob2:
text1.append(buchung1[-1])
text1.append(buchung2[-1])
else:
text1.append("# " + buchung1[-1])
text1.append("# " + buchung2[-1])
text1.append(zeile1)
text1.append(zeile2)
else:
text.append(buchung1[-1])
text.append(buchung2[-1])
buchungen.pop()
buchungen.remove(buchung)
break
for buchung in buchungen:
text.append(buchung[-1])
ktotext = "\n".join(text) + "\n" + "\n".join(text1) + "\n"
open(file,"w").write(ktotext)