From 32dd89caead0dff1c8f23c3535cd357f814bb9a9 Mon Sep 17 00:00:00 2001
From: Constantin Wenger <constantin.wenger@googlemail.com>
Date: Thu, 13 Jun 2019 11:12:44 +0000
Subject: [PATCH] fixed some things and made hash generation and image download parallel also added set icon download (it needs the sets.json from scryfall in data dir)
---
fetch_data.py | 7 ++++++-
1 files changed, 6 insertions(+), 1 deletions(-)
diff --git a/fetch_data.py b/fetch_data.py
index 00bf592..500dc55 100644
--- a/fetch_data.py
+++ b/fetch_data.py
@@ -81,9 +81,14 @@
# df is a single row of card
fetch_card_image(df, out_dir, size)
else:
+ from concurrent.futures import ThreadPoolExecutor, wait as fwait
+ executor = ThreadPoolExecutor(5)
# df is a dataframe containing list of cards
+ arglist = []
for ind, row in df.iterrows():
- fetch_card_image(row, out_dir, size)
+ arglist.append(executor.submit(fetch_card_image, row, out_dir, size))
+ fwait(arglist)
+ # fetch_card_image(row, out_dir, size)
def fetch_card_image(row, out_dir=None, size='png'):
--
Gitblit v1.10.0