I Have a program which gets an input from RabbitMQ's queue and then run a scrapy spider passing this input. It works fine, but just for the first time, after the first input it throws this error:
twisted.internet.error.ReactorNotRestartable
Here is my code:
def start_flow(crawler_class):
# Create the queue and all the connections
queue = Queue(connection_uri='amqp://guest:guest@localhost:5672/')
queue.create_connection()
queue.create_channel()
queue_name = _get_queue_name(crawler_class)
queue.define_queue(queue_name)
# ---------------------------------------
while True:
print("Waiting for the next input")
# Get an input
queue.consume(queue_name)
queue.channel.start_consuming()
input_ = queue.get_next_input()
# ------------
# Runs the crawler passing the input
runner = CrawlerRunner()
d = runner.crawl(crawler_class, input=input_)
d.addCallback(lambda _: reactor.stop())
reactor.run()
# ----------------------------
I already tried CrawlerProcess, but it throws the same problem