From b88a7221000facda2fd81330417958adcc17dc3c Mon Sep 17 00:00:00 2001
From: Edmond Yoo <hj3yoo@uwaterloo.ca>
Date: Sun, 26 Aug 2018 02:22:57 +0000
Subject: [PATCH] Aggregating card images, importing texture images for background
---
fetch_data.py | 36 ++++++++++++++++++++++++------------
1 files changed, 24 insertions(+), 12 deletions(-)
diff --git a/fetch_data.py b/fetch_data.py
index 5e3f23c..6fc526a 100644
--- a/fetch_data.py
+++ b/fetch_data.py
@@ -2,12 +2,14 @@
import json
import pandas as pd
import re
+import os
def fetch_all_cards_text(url='https://api.scryfall.com/cards/search?q=layout:normal+format:modern+lang:en+frame:2003',
csv_name=''):
has_more = True
cards = []
+ # get cards dataset as a json from the query
while has_more:
res_file_dir, http_message = request.urlretrieve(url)
with open(res_file_dir) as res_file:
@@ -18,23 +20,21 @@
url = res_json['next_page']
print(len(cards))
+ # Convert them into a dataframe, and truncate unnecessary columns
df = pd.DataFrame.from_dict(cards)
- df['image'] = ''
- for ind, row in df.iterrows():
- df.set_value(ind, 'image', row['image_uris']['png'])
if csv_name != '':
df = df[['artist', 'border_color', 'collector_number', 'color_identity', 'colors', 'flavor_text', 'image_uris',
- 'image', 'mana_cost', 'legalities', 'name', 'oracle_text', 'rarity', 'type_line', 'set', 'set_name',
- 'power', 'toughness']]
- df.to_csv(csv_name, sep=';')
+ 'mana_cost', 'legalities', 'name', 'oracle_text', 'rarity', 'type_line', 'set', 'set_name', 'power',
+ 'toughness']]
+ df.to_csv(csv_name, sep=';') # Comma doesn't work, since some columns are saved as a dict
- return cards
+ return df
+# Pulled from Django framework (https://github.com/django/django/blob/master/django/utils/text.py)
def get_valid_filename(s):
"""
- NOTE: Pulled from Django framework (https://github.com/django/django/blob/master/django/utils/text.py)
Return the given string converted to a string that can be used for a clean
filename. Remove leading and trailing spaces; convert other spaces to
underscores; and remove anything that is not an alphanumeric, dash,
@@ -46,14 +46,26 @@
return re.sub(r'(?u)[^-\w.]', '', s)
-def fetch_cards_image(cards_json, out_dir, size='large'):
- for card in cards_json:
- request.urlretrieve(card['image_uris'][size], '%s\%s' % (out_dir, card['name']))
+def fetch_cards_image(df, out_dir='', size='png'):
+ for ind, row in df.iterrows():
+ png_url = row['image_uris'][size]
+ if out_dir == '':
+ out_dir = 'data/png/%s' % row['set']
+ if not os.path.exists(out_dir):
+ os.makedirs(out_dir)
+ img_name = '%s/%s_%s.png' % (out_dir, row['collector_number'], get_valid_filename(row['name']))
+ request.urlretrieve(png_url, filename=img_name)
+ print(img_name)
pass
def main():
- fetch_all_cards_text(csv_name='data/all_cards.csv')
+ df = fetch_all_cards_text(url='https://api.scryfall.com/cards/search?q=layout:normal+set:rtr+lang:en',
+ csv_name='data/all_cards.csv')
+ fetch_cards_image(df)
+ pass
+
if __name__ == '__main__':
main()
+ pass
--
Gitblit v1.10.0