I`m making the multithread system on python 2.7. Basically, it has 3 thread and one singleton-class with shared data.
Every thread is separate class in a file. The file main.py import working and communication files, and shared data. Then main thread invoke working class in one thread and communication in another one thread. Herewith shared data, as only one instance of singleton, is passed in constructors of working class and communication class.
File main.py
import communication
import Worker
import Data
app_data = Data.Instance()
#...........
SRV = communication.Server(app_data)
SRV.setDaemon(True)
SRV.start()
#...........
while True
#...........
# MUST BE locker.acquire()
if condition1:
if condition2:
job = Worker(app_data, SRV.taskResultSendToSlaves, app_data.ip_table[app_data.cfg.MY_IP]['tasks'].pop())
job.setDaemon(True)
job.start()
# MUST BE locker.release()
File communication.py
class Server(threading.Thread):
# .................
def __init__(self, data):
self.data = data
# .................
threading.Thread.__init__(self)
def run(self):
srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
srv.settimeout(self.data.cfg.TIMEOUT)
srv.bind((self.my_addr, self.my_port))
srv.listen(self.data.cfg.NUMBER_OF_CLIENTS)
print "Start server"
while True:
# HANDLING MESSAGES FROM OTHER PC
# .................
File Worker.py
class Worker(threading.Thread):
def __init__(self, data, sender, taskname):
self.data = data
self.sender = sender
self.taskname = taskname
threading.Thread.__init__(self)
def run(self):
import thread
self.data.complete_task.clear()
tick_before = time.time()
startupinfo = subprocess.STARTUPINFO()
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
startupinfo.wShowWindow = subprocess.SW_HIDE
p = subprocess.Popen(self.data.cfg.PATH_INTERPRETER + " " + self.data.cfg.PATH_TASKS + self.taskname, startupinfo=startupinfo, shell=False, stdout=subprocess.PIPE)
job_result, err = p.communicate()
tick_after = time.time()
work_time = tick_after - tick_before
# MUST BE locker.acquire()
self.data.task_table[self.taskname]['status'] = 'complete'
self.data.task_table[self.taskname]['result'] = job_result
self.data.task_table[self.taskname]['time'] = work_time
# MUST BE locker.release()
logging.debug("%s task is done" % self.taskname)
tr = threading.Thread(target=self.sender, name="SENDER", args=(self.taskname, ))
tr.setDaemon(True)
tr.start()
tr.join()
logging.debug("%s task is sent" % self.taskname)
self.data.complete_task.set()
thread.exit()
Singletone.py
class Singleton:
def __init__(self, decorated):
self._decorated = decorated
def Instance(self):
try:
return self._instance
except AttributeError:
self._instance = self._decorated()
return self._instance
def __call__(self):
raise TypeError('Singletons must be accessed through `Instance()`.')
def __instancecheck__(self, inst):
return isinstance(inst, self._decorated)
data.py
#-*- coding: utf-8 -*-
from singletone import Singleton
from configs import Configurations
import threading
import logging
@Singleton
class Data:
def __init__(self):
logging.basicConfig(format=u'%(filename)-10s[LINE:%(lineno)d] <%(funcName)-15s> # %(levelname)-8s [%(asctime)s] %(message)s'.encode('cp1251', 'ignore'), level=logging.DEBUG, filename='mylog.log')
logging.log(100, '='*120)
self.cfg = Configurations()
self.ip_table = self.getIPTable()
self.task_table = self.getTaskTable()
self.locker = threading.Lock()
self.initialization = threading.Event()
self.initialization.clear()
self.identification = threading.Event()
self.identification.clear()
self.complete_task = threading.Event()
self.complete_task.set()
self.flag_of_close = False
def __str__(self):
return "\
{0}\n\
\n\
{1}\n\
\n\
{2}\n\
".format(str(self.cfg), self.strIPTable(), self.strTaskTable())
def strIPTable(self):
#return str(self.ip_table)
result = ["%s = %s" % (key, str(value)) for key, value in self.ip_table.items()]
result.sort()
return "\n\t\t".join(result)
def strTaskTable(self):
#return str(self.task_table)
result = ["%s = %s" % (key, str(value)) for key, value in self.task_table.items()]
result.sort()
return "\n\t\t".join(result)
def getIPTable(self):
result = {}
if self.cfg.IPS:
result = dict((item.strip(), {'status': True, 'port': 8000, 'tasks': []}) for item in self.cfg.IPS.split(','))
# result = dict((item.strip(), {'status': False, 'port': 8000, 'tasks': []}) for item in self.cfg.IPS.split(','))
result[self.cfg.MY_IP] = {'status': True, 'port': 8000, 'tasks': []}
return result
def getTaskTable(self):
result = {}
if self.cfg.TASKS:
result = dict((item.strip(), {'status': 'uncomplete', 'result': '', 'time': 0}) for item in self.cfg.TASKS.split(','))
return result
def getTotalCompleteTasks(self):
result = 0
for taskname in self.task_table.keys():
if self.task_table[taskname]['status'] == 'complete':
result += 1
return result
if __name__ == '__main__':
data = Data.Instance()
print data
Singleton i has stolen from stackoverflow
After start this system, sometimes a have a data race. When working and main thread at the same time read shared data. I think we need a threading.Lock here. Then i did a mistake, i put Lock object in shared data and have used it to separate access. Shortly a have understand my mistake.
Filenames has changed, some pieces of code has removed.
But for now i don't know where i must put Lock object, that every thread can easily and in a right way access and use it. Can you give me advice?
My english is not very good, so be tolerant. I hope you understand my question...
P.S.
Besides that, i tried to pass Lock() object in constructors of classes. And i have same trouble. The application has fallen somewhere, where data was accessed. And i can't find out where exactly it is. Every start can fall down application with 50% probability.
