From 8468f19b741475059bbd0bc4db310ee05ea7e6fb Mon Sep 17 00:00:00 2001
From: SpeedProg <speedprog@googlemail.com>
Date: Thu, 02 Jan 2020 16:05:16 +0000
Subject: [PATCH] removed test files
---
fetch_data.py | 17 ++++++++++++++---
1 files changed, 14 insertions(+), 3 deletions(-)
diff --git a/fetch_data.py b/fetch_data.py
index 00bf592..e765ef3 100644
--- a/fetch_data.py
+++ b/fetch_data.py
@@ -81,9 +81,14 @@
# df is a single row of card
fetch_card_image(df, out_dir, size)
else:
+ from concurrent.futures import ThreadPoolExecutor, wait as fwait
+ executor = ThreadPoolExecutor(5)
# df is a dataframe containing list of cards
+ arglist = []
for ind, row in df.iterrows():
- fetch_card_image(row, out_dir, size)
+ arglist.append(executor.submit(fetch_card_image, row, out_dir, size))
+ fwait(arglist)
+ # fetch_card_image(row, out_dir, size)
def fetch_card_image(row, out_dir=None, size='png'):
@@ -95,7 +100,10 @@
:return:
"""
if out_dir is None:
- out_dir = '%s/card_img/%s/%s' % (Config.data_dir, size, row['set'])
+ s = row['set']
+ if s == 'con':
+ s = 'con__'
+ out_dir = '%s/card_img/%s/%s' % (Config.data_dir, size, s)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
@@ -128,7 +136,10 @@
def main():
# Query card data by each set, then merge them together
for set_name in Config.all_set_list:
- csv_name = '%s/csv/%s.csv' % (Config.data_dir, set_name)
+ set_name_m = set_name
+ if set_name_m == 'con':
+ set_name_m = 'con__'
+ csv_name = '%s/csv/%s.csv' % (Config.data_dir, set_name_m)
print(csv_name)
if not os.path.isfile(csv_name):
df = fetch_all_cards_text(url='https://api.scryfall.com/cards/search?q=set:%s+lang:en' % set_name,
--
Gitblit v1.10.0