From c59db54775606349f6ba5c6cab7fcb34498bb31d Mon Sep 17 00:00:00 2001
From: Edmond Yoo <hj3yoo@uwaterloo.ca>
Date: Sat, 13 Oct 2018 18:18:42 +0000
Subject: [PATCH] Cleaning & commenting #2 - updating comments & docstrings
---
fetch_data.py | 95 ++++++++++++++++++++++++++++++++---------------
1 files changed, 64 insertions(+), 31 deletions(-)
diff --git a/fetch_data.py b/fetch_data.py
index dcfc732..4f36181 100644
--- a/fetch_data.py
+++ b/fetch_data.py
@@ -5,26 +5,40 @@
import re
import os
import transform_data
-import time
-all_set_list = [# Core & expansion sets with 2003 frame
- 'mrd', 'dst', '5dn', 'chk', 'bok', 'sok', 'rav', 'gpt', 'dis', 'csp', 'tsp', 'plc', 'fut', '10e', 'lrw',
- 'mor', 'shm', 'eve', 'ala', 'con', 'arb', 'm10', 'zen', 'wwk', 'roe', 'm11', 'som', 'mbs', 'nph', 'm12',
- 'isd', 'dka', 'avr', 'm13', 'rtr', 'gtc', 'dgm', 'm14', 'ths', 'bng', 'jou',
- # Core & expansion sets with 2015 frame
- 'm15', 'ktk', 'frf', 'dtk', 'bfz', 'ogw', 'soi', 'emn', 'kld', 'aer', 'akh', 'hou', 'xln', 'rix', 'dom',
- # Box sets
- 'evg', 'drb', 'dd2', 'ddc', 'td0', 'v09', 'ddd', 'h09', 'dde', 'dpa', 'v10', 'ddf', 'td0', 'pd2', 'ddg',
+"""
+Note: All codes in this file realies on Scryfall API to aggregate card database and their images.
+Scryfall API doc is available at: https://scryfall.com/docs/api
+"""
+
+# List of all black-bordered cards printed from 8th edition and onwards (8ed and 9ed are white-bordered)
+# Core & expansion sets with 2003 frame
+set_2003_list = ['mrd', 'dst', '5dn', 'chk', 'bok', 'sok', 'rav', 'gpt', 'dis', 'csp', 'tsp', 'plc', 'fut', '10e',
+ 'lrw', 'mor', 'shm', 'eve', 'ala', 'con', 'arb', 'm10', 'zen', 'wwk', 'roe', 'm11', 'som', 'mbs',
+ 'nph', 'm12', 'isd', 'dka', 'avr', 'm13', 'rtr', 'gtc', 'dgm', 'm14', 'ths', 'bng', 'jou']
+# Core & expansion sets with 2015 frame
+set_2015_list = ['m15', 'ktk', 'frf', 'dtk', 'bfz', 'ogw', 'soi', 'emn', 'kld', 'aer', 'akh', 'hou', 'xln', 'rix', 'dom']
+
+# Box sets
+set_box_list = ['evg', 'drb', 'dd2', 'ddc', 'td0', 'v09', 'ddd', 'h09', 'dde', 'dpa', 'v10', 'ddf', 'td0', 'pd2', 'ddg',
'cmd', 'v11', 'ddh', 'pd3', 'ddi', 'v12', 'ddj', 'cm1', 'td2', 'ddk', 'v13', 'ddl', 'c13', 'ddm', 'md1',
'v14', 'ddn', 'c14', 'ddo', 'v15', 'ddp', 'c15', 'ddq', 'v16', 'ddr', 'c16', 'pca', 'dds', 'cma', 'c17',
- 'ddt', 'v17', 'ddu', 'cm2', 'ss1', 'gs1', 'c18',
- # Supplemental sets
- 'HOP', 'ARC', 'PC2', 'CNS', 'CN2', 'E01', 'E02', 'BBD'
- ]
+ 'ddt', 'v17', 'ddu', 'cm2', 'ss1', 'gs1', 'c18']
+
+# Supplemental sets
+set_sup_list = ['hop', 'arc', 'pc2', 'cns', 'cn2', 'e01', 'e02', 'bbd']
+
+all_set_list = set_2003_list
def fetch_all_cards_text(url='https://api.scryfall.com/cards/search?q=layout:normal+format:modern+lang:en+frame:2003',
- csv_name=''):
+ csv_name=None):
+ """
+ Given the query URL using Scryfall API, aggregate all card information and convert them from json to table
+ :param url: query URL
+ :param csv_name: path of the csv file to save the result
+ :return: pandas dataframe of the fetch cards
+ """
has_more = True
cards = []
# get cards dataset as a json from the query
@@ -41,25 +55,19 @@
# Convert them into a dataframe, and truncate unnecessary columns
df = pd.DataFrame.from_dict(cards)
- if csv_name != '':
+ if csv_name is not None:
#df = df[['artist', 'border_color', 'collector_number', 'color_identity', 'colors', 'flavor_text', 'image_uris',
# 'mana_cost', 'legalities', 'name', 'oracle_text', 'rarity', 'type_line', 'set', 'set_name', 'power',
# 'toughness']]
- #df.to_json(csv_name)
- df.to_csv(csv_name, sep=';') # Comma doesn't work, since some columns are saved as a dict
-
+ df.to_csv(csv_name, sep=';') # Comma seperator doesn't work, since some columns are saved as a dict
return df
def load_all_cards_text(csv_name):
- #with open(csv_name, 'r') as json_file:
- # cards = json.loads(json_file.read())
- #df = pd.DataFrame.from_dict(cards)
- df = pd.read_csv(csv_name, sep=';')
+ df = pd.read_csv(csv_name, sep=';') # Comma seperator doesn't work, since some columns are saved as a dict
return df
-# Pulled from Django framework (https://github.com/django/django/blob/master/django/utils/text.py)
def get_valid_filename(s):
"""
Return the given string converted to a string that can be used for a clean
@@ -68,28 +76,52 @@
underscore, or dot.
>>> get_valid_filename("john's portrait in 2004.jpg")
'johns_portrait_in_2004.jpg'
+ From: https://github.com/django/django/blob/master/django/utils/text.py
+ :param s: input string
+ :return: string of valid filename
"""
s = str(s).strip().replace(' ', '_')
return re.sub(r'(?u)[^-\w.]', '', s)
-def fetch_all_cards_image(df, out_dir='', size='png'):
+def fetch_all_cards_image(df, out_dir=None, size='png'):
+ """
+ Download card images from Scryfall database
+ :param df: pandas dataframe (or series) of cards
+ :param out_dir: path of output directory
+ :param size: Image format given by Scryfall API - 'png', 'large', 'normal', 'small', 'art_crop', 'border_crop'
+ :return:
+ """
+ if size != 'png':
+ print('Note: this repo has been implemented using only \'png\' size. '
+ 'Using %s may result in an unexpected behaviour in other parts of this repo.' % size)
if isinstance(df, pd.Series):
+ # df is a single row of card
fetch_card_image(df, out_dir, size)
else:
+ # df is a dataframe containing list of cards
for ind, row in df.iterrows():
fetch_card_image(row, out_dir, size)
-def fetch_card_image(row, out_dir='', size='png'):
- if out_dir == '':
+def fetch_card_image(row, out_dir=None, size='png'):
+ """
+ Download a card's image from Scryfall database
+ :param row: pandas series including the card's information
+ :param out_dir: path of the output directory
+ :param size: Image format given by Scryfall API - 'png', 'large', 'normal', 'small', 'art_crop', 'border_crop'
+ :return:
+ """
+ if out_dir is None:
out_dir = '%s/card_img/%s/%s' % (transform_data.data_dir, size, row['set'])
if not os.path.exists(out_dir):
os.makedirs(out_dir)
+ # Extract card's name and URL for image accordingly
+ # Double-faced cards have a different format, and results in two separate card images
png_urls = []
card_names = []
- if row['layout'] == 'transform' or row['layout'] == 'double_faced_token':
+ if row['layout'] in ['transform', 'double_faced_token']:
if isinstance(row['card_faces'], str): # For some reason, dict isn't being parsed in the previous step
card_faces = ast.literal_eval(row['card_faces'])
else:
@@ -112,21 +144,22 @@
def main():
+ # Query card data by each set, then merge them together
for set_name in all_set_list:
csv_name = '%s/csv/%s.csv' % (transform_data.data_dir, set_name)
print(csv_name)
if not os.path.isfile(csv_name):
- df = fetch_all_cards_text(url='https://api.scryfall.com/cards/search?q=set:%s+lang:en'
- % set_name, csv_name=csv_name)
+ df = fetch_all_cards_text(url='https://api.scryfall.com/cards/search?q=set:%s+lang:en' % set_name,
+ csv_name=csv_name)
else:
df = load_all_cards_text(csv_name)
df.sort_values('collector_number')
fetch_all_cards_image(df, out_dir='%s/card_img/png/%s' % (transform_data.data_dir, set_name))
+
#df = fetch_all_cards_text(url='https://api.scryfall.com/cards/search?q=layout:normal+lang:en+frame:2003',
# csv_name='data/csv/all.csv')
- pass
+ return
if __name__ == '__main__':
main()
- pass
--
Gitblit v1.10.0