From b7dd760578dbbde908c6779782a91cfcbc916d38 Mon Sep 17 00:00:00 2001
From: Constantin Wenger <constantin.wenger@googlemail.com>
Date: Fri, 21 Jun 2019 10:36:10 +0000
Subject: [PATCH] extra set detection code
---
fetch_data.py | 43 +++++++++++++++----------------------------
1 files changed, 15 insertions(+), 28 deletions(-)
diff --git a/fetch_data.py b/fetch_data.py
index 4f36181..500dc55 100644
--- a/fetch_data.py
+++ b/fetch_data.py
@@ -1,35 +1,17 @@
-from urllib import request, error
import ast
import json
+import os
import pandas as pd
import re
-import os
-import transform_data
+from urllib import request, error
+
+from config import Config
"""
Note: All codes in this file realies on Scryfall API to aggregate card database and their images.
Scryfall API doc is available at: https://scryfall.com/docs/api
"""
-# List of all black-bordered cards printed from 8th edition and onwards (8ed and 9ed are white-bordered)
-# Core & expansion sets with 2003 frame
-set_2003_list = ['mrd', 'dst', '5dn', 'chk', 'bok', 'sok', 'rav', 'gpt', 'dis', 'csp', 'tsp', 'plc', 'fut', '10e',
- 'lrw', 'mor', 'shm', 'eve', 'ala', 'con', 'arb', 'm10', 'zen', 'wwk', 'roe', 'm11', 'som', 'mbs',
- 'nph', 'm12', 'isd', 'dka', 'avr', 'm13', 'rtr', 'gtc', 'dgm', 'm14', 'ths', 'bng', 'jou']
-# Core & expansion sets with 2015 frame
-set_2015_list = ['m15', 'ktk', 'frf', 'dtk', 'bfz', 'ogw', 'soi', 'emn', 'kld', 'aer', 'akh', 'hou', 'xln', 'rix', 'dom']
-
-# Box sets
-set_box_list = ['evg', 'drb', 'dd2', 'ddc', 'td0', 'v09', 'ddd', 'h09', 'dde', 'dpa', 'v10', 'ddf', 'td0', 'pd2', 'ddg',
- 'cmd', 'v11', 'ddh', 'pd3', 'ddi', 'v12', 'ddj', 'cm1', 'td2', 'ddk', 'v13', 'ddl', 'c13', 'ddm', 'md1',
- 'v14', 'ddn', 'c14', 'ddo', 'v15', 'ddp', 'c15', 'ddq', 'v16', 'ddr', 'c16', 'pca', 'dds', 'cma', 'c17',
- 'ddt', 'v17', 'ddu', 'cm2', 'ss1', 'gs1', 'c18']
-
-# Supplemental sets
-set_sup_list = ['hop', 'arc', 'pc2', 'cns', 'cn2', 'e01', 'e02', 'bbd']
-
-all_set_list = set_2003_list
-
def fetch_all_cards_text(url='https://api.scryfall.com/cards/search?q=layout:normal+format:modern+lang:en+frame:2003',
csv_name=None):
@@ -99,9 +81,14 @@
# df is a single row of card
fetch_card_image(df, out_dir, size)
else:
+ from concurrent.futures import ThreadPoolExecutor, wait as fwait
+ executor = ThreadPoolExecutor(5)
# df is a dataframe containing list of cards
+ arglist = []
for ind, row in df.iterrows():
- fetch_card_image(row, out_dir, size)
+ arglist.append(executor.submit(fetch_card_image, row, out_dir, size))
+ fwait(arglist)
+ # fetch_card_image(row, out_dir, size)
def fetch_card_image(row, out_dir=None, size='png'):
@@ -113,7 +100,7 @@
:return:
"""
if out_dir is None:
- out_dir = '%s/card_img/%s/%s' % (transform_data.data_dir, size, row['set'])
+ out_dir = '%s/card_img/%s/%s' % (Config.data_dir, size, row['set'])
if not os.path.exists(out_dir):
os.makedirs(out_dir)
@@ -145,8 +132,8 @@
def main():
# Query card data by each set, then merge them together
- for set_name in all_set_list:
- csv_name = '%s/csv/%s.csv' % (transform_data.data_dir, set_name)
+ for set_name in Config.all_set_list:
+ csv_name = '%s/csv/%s.csv' % (Config.data_dir, set_name)
print(csv_name)
if not os.path.isfile(csv_name):
df = fetch_all_cards_text(url='https://api.scryfall.com/cards/search?q=set:%s+lang:en' % set_name,
@@ -154,10 +141,10 @@
else:
df = load_all_cards_text(csv_name)
df.sort_values('collector_number')
- fetch_all_cards_image(df, out_dir='%s/card_img/png/%s' % (transform_data.data_dir, set_name))
+ fetch_all_cards_image(df, out_dir='%s/card_img/png/%s' % (Config.data_dir, set_name))
#df = fetch_all_cards_text(url='https://api.scryfall.com/cards/search?q=layout:normal+lang:en+frame:2003',
- # csv_name='data/csv/all.csv')
+ # csv_name='%s/csv/all.csv' % Config.data_dir)
return
--
Gitblit v1.10.0