Python thread class wont exit

July 27, 2022

I want to exit the crawler script which is using threading, but it doesn’t detect Ctrl+c. I tried to try and except inside the main thread and in newly created threads, but it doesn’t raise the exception, signal in main thread, but main thread isn’t active, and it doesn’t detect it.

import threading
import signal

class Crawler(threading.Thread):
    def __init__(self, num):
        threading.Thread.__init__(self)
        # Other stuff

    def run(self, ):
        try:
            self.crawl()
        except KeyboardInterrupt:
            self.exit()
    
    def crawl(self):
        for url in self.links_list:
            try:
                # Getting the website and all links on it and adding it to links_list
                pass
                
            except (KeyboardInterrupt, SystemExit):
                self.exit()

            except AssertionError:
                self.exit()
    
    def exit(self):
        # Saving progress
        raise SystemExit()

def handler(signum, frame):
    exit()

if __name__ == "__main__":
    signal.signal(signal.SIGINT, handler)
    for i in range(8):
        crawler = Crawler(i)
        crawler.start()

>Solution :

Instead of trying to do stuff with signals, use a threading.Event() to signal all of the other threads to stop:

import threading
import time

stop_signal = threading.Event()


class Crawler(threading.Thread):
    def __init__(self, num):
        threading.Thread.__init__(self)
        # Other stuff

    def run(self):
        try:
            for url in self.links_list:
                # Between every url, see if we should stop
                if stop_signal.is_set():
                    break
                # Getting the website and all links on it and adding it to links_list
        finally:
            self.save_progress()

    def save_progress(self):
        pass


def main():
    crawlers = [Crawler(x) for x in range(8)]
    for crawler in crawlers:
        crawler.start()

    # Do nothing for a whole long while in the main loop
    # (you could exit the loop when all jobs are done, though)

    while True:
        try:
            time.sleep(1)
        except KeyboardInterrupt:
            stop_signal.set()
            break
    for crawler in crawlers:
        crawler.join()


if __name__ == "__main__":
    main()