def run(self):
while True:
self.url = self.queue.get()
#put working list to tell worker thread not to put url twice
self.work_locker.acquire()
self.work_list.append(self.url)
self.work_locker.release()
print(\'miner is running : \' + self.getName() +\' has work with \' + self.url )
#try:
#build cache
#get post queue first
self.__parser_list.ParseThis(self.url)
self.__temp_post_queue = self.__parser_list.GetQueue()
#print(self.__temp_post_queue.qsize())
#get processed post
self.__temp_postdata = queue.Queue()
for a in range(2):
self.__t = gall_parser.post_parser(self.__temp_post_queue, self.__temp_postdata)
self.__t.setDaemon(True)
self.__t.start()
#wait until post queue become empty
self.__temp_post_queue.join()
#print(\"post output data number\" + str(self.__temp_postdata.qsize()) )
#sort by time order
self.__temp_post_list = []
while not self.__temp_postdata.empty() :
elem = self.__temp_postdata.get()
#print(elem)
#for elem in self.__temp_postdata:
self.__temp_post_list.append(elem)
#[url, order]
self.__temp_post_list.sort( key=operator.itemgetter(1) )
#make the cache
self.__temp_cache = page_maker(self.url, self.__temp_post_list)
#add a cache in list
#lock cache list
self.locker.acquire()
#cache : [url, time of using, time, data]
self.__status_adding = False
#find if there is the old one
for self.__cache_elem in self.cache_list:
if self.__cache_elem[0] == self.url:
self.__cache_elem[2] = time.time()
self.__cache_elem[3] = self.__temp_cache
self.__status_adding = True
print(\"old cache has been refreshed\")
break;
#There was no old one
if not self.__status_adding:
#check whether cache list reach the max number
if len(self.cache_list) >= self.max_cache:
#find the least used cache
self.__temp_least_used = self.cache_list[0][1]
for self.__cache_elem in self.cache_list:
if self.__temp_least_used < self.__cache_elem[1]:
self.__temp_least_used = self.__cache_elem[1]
#replace the least used one
self.__cache_elem[0] = self.url
self.__cache_elem[1] = 0
self.__cache_elem[2] = time.time()
self.__cache_elem[3] = self.__temp_cache
#if cache list isn\'t full
else:
self.cache_list.append( [self.url, 0, time.time(), self.__temp_cache] )
#unlock cache list
self.locker.release()
#work completed, put off item in working list
self.work_locker.acquire()
try:
self.work_list.remove(self.url)
except:
print(\"What happened!?\")
self.work_locker.release()
주석 이정도 달면 괜찮음?
while True:
self.url = self.queue.get()
#put working list to tell worker thread not to put url twice
self.work_locker.acquire()
self.work_list.append(self.url)
self.work_locker.release()
print(\'miner is running : \' + self.getName() +\' has work with \' + self.url )
#try:
#build cache
#get post queue first
self.__parser_list.ParseThis(self.url)
self.__temp_post_queue = self.__parser_list.GetQueue()
#print(self.__temp_post_queue.qsize())
#get processed post
self.__temp_postdata = queue.Queue()
for a in range(2):
self.__t = gall_parser.post_parser(self.__temp_post_queue, self.__temp_postdata)
self.__t.setDaemon(True)
self.__t.start()
#wait until post queue become empty
self.__temp_post_queue.join()
#print(\"post output data number\" + str(self.__temp_postdata.qsize()) )
#sort by time order
self.__temp_post_list = []
while not self.__temp_postdata.empty() :
elem = self.__temp_postdata.get()
#print(elem)
#for elem in self.__temp_postdata:
self.__temp_post_list.append(elem)
#[url, order]
self.__temp_post_list.sort( key=operator.itemgetter(1) )
#make the cache
self.__temp_cache = page_maker(self.url, self.__temp_post_list)
#add a cache in list
#lock cache list
self.locker.acquire()
#cache : [url, time of using, time, data]
self.__status_adding = False
#find if there is the old one
for self.__cache_elem in self.cache_list:
if self.__cache_elem[0] == self.url:
self.__cache_elem[2] = time.time()
self.__cache_elem[3] = self.__temp_cache
self.__status_adding = True
print(\"old cache has been refreshed\")
break;
#There was no old one
if not self.__status_adding:
#check whether cache list reach the max number
if len(self.cache_list) >= self.max_cache:
#find the least used cache
self.__temp_least_used = self.cache_list[0][1]
for self.__cache_elem in self.cache_list:
if self.__temp_least_used < self.__cache_elem[1]:
self.__temp_least_used = self.__cache_elem[1]
#replace the least used one
self.__cache_elem[0] = self.url
self.__cache_elem[1] = 0
self.__cache_elem[2] = time.time()
self.__cache_elem[3] = self.__temp_cache
#if cache list isn\'t full
else:
self.cache_list.append( [self.url, 0, time.time(), self.__temp_cache] )
#unlock cache list
self.locker.release()
#work completed, put off item in working list
self.work_locker.acquire()
try:
self.work_list.remove(self.url)
except:
print(\"What happened!?\")
self.work_locker.release()
주석 이정도 달면 괜찮음?
댓글 0