From 60268dea8608567ecb3f7fe4132b6bc687a32c64 Mon Sep 17 00:00:00 2001
From: Constantin Wenger <constantin.wenger@googlemail.com>
Date: Tue, 13 Aug 2019 20:42:38 +0000
Subject: [PATCH] scaled min size threshold down to 10k again from 100k because it won't work properly with lower res webcams
---
fetch_data.py | 17 ++++++++++++++---
1 files changed, 14 insertions(+), 3 deletions(-)
diff --git a/fetch_data.py b/fetch_data.py
index 00bf592..e765ef3 100644
--- a/fetch_data.py
+++ b/fetch_data.py
@@ -81,9 +81,14 @@
# df is a single row of card
fetch_card_image(df, out_dir, size)
else:
+ from concurrent.futures import ThreadPoolExecutor, wait as fwait
+ executor = ThreadPoolExecutor(5)
# df is a dataframe containing list of cards
+ arglist = []
for ind, row in df.iterrows():
- fetch_card_image(row, out_dir, size)
+ arglist.append(executor.submit(fetch_card_image, row, out_dir, size))
+ fwait(arglist)
+ # fetch_card_image(row, out_dir, size)
def fetch_card_image(row, out_dir=None, size='png'):
@@ -95,7 +100,10 @@
:return:
"""
if out_dir is None:
- out_dir = '%s/card_img/%s/%s' % (Config.data_dir, size, row['set'])
+ s = row['set']
+ if s == 'con':
+ s = 'con__'
+ out_dir = '%s/card_img/%s/%s' % (Config.data_dir, size, s)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
@@ -128,7 +136,10 @@
def main():
# Query card data by each set, then merge them together
for set_name in Config.all_set_list:
- csv_name = '%s/csv/%s.csv' % (Config.data_dir, set_name)
+ set_name_m = set_name
+ if set_name_m == 'con':
+ set_name_m = 'con__'
+ csv_name = '%s/csv/%s.csv' % (Config.data_dir, set_name_m)
print(csv_name)
if not os.path.isfile(csv_name):
df = fetch_all_cards_text(url='https://api.scryfall.com/cards/search?q=set:%s+lang:en' % set_name,
--
Gitblit v1.10.0