From be11e02f1d08beb40fff7d54d1c7a343ba10b275 Mon Sep 17 00:00:00 2001
From: Constantin Wenger <constantin.wenger@googlemail.com>
Date: Tue, 01 Feb 2022 22:46:28 +0000
Subject: [PATCH] added autocreating csv folder
---
fetch_data.py | 165 ++++++++++++++++++++++++++++++++++++++++++++++++++++--
1 files changed, 157 insertions(+), 8 deletions(-)
diff --git a/fetch_data.py b/fetch_data.py
index 190dd14..a43cb68 100644
--- a/fetch_data.py
+++ b/fetch_data.py
@@ -1,11 +1,160 @@
-from urllib import request
+import ast
+import json
+import os
+import pandas as pd
+import re
+from urllib import request, error
+
+from config import Config
+
+"""
+Note: All codes in this file realies on Scryfall API to aggregate card database and their images.
+Scryfall API doc is available at: https://scryfall.com/docs/api
+"""
-def fetch_all_cards_text(url='https://archive.scryfall.com/json/scryfall-all-cards.json', out_file='all_cards.json'):
- request.urlretrieve(url, out_file)
- pass
+def fetch_all_cards_text(url='https://api.scryfall.com/cards/search?q=layout:normal+format:modern+lang:en+frame:2003',
+ csv_name=None):
+ """
+ Given the query URL using Scryfall API, aggregate all card information and convert them from json to table
+ :param url: query URL
+ :param csv_name: path of the csv file to save the result
+ :return: pandas dataframe of the fetch cards
+ """
+ has_more = True
+ cards = []
+ # get cards dataset as a json from the query
+ while has_more:
+ res_file_dir, http_message = request.urlretrieve(url)
+ with open(res_file_dir, 'r') as res_file:
+ res_json = json.loads(res_file.read())
+ cards += res_json['data']
+ has_more = res_json['has_more']
+ if has_more:
+ url = res_json['next_page']
+ print(len(cards))
-def fetch_cards_image(cards_json, out_dir, size='large'):
- for card in cards_json:
- request.urlretrieve(card['image_uris'][size], '%s\%s' % (out_dir, card['name']))
- pass
\ No newline at end of file
+ # Convert them into a dataframe, and truncate unnecessary columns
+ df = pd.DataFrame.from_dict(cards)
+
+ if csv_name is not None:
+ #df = df[['artist', 'border_color', 'collector_number', 'color_identity', 'colors', 'flavor_text', 'image_uris',
+ # 'mana_cost', 'legalities', 'name', 'oracle_text', 'rarity', 'type_line', 'set', 'set_name', 'power',
+ # 'toughness']]
+ df.to_csv(csv_name, sep=';') # Comma seperator doesn't work, since some columns are saved as a dict
+ return df
+
+
+def load_all_cards_text(csv_name):
+ df = pd.read_csv(csv_name, sep=';') # Comma seperator doesn't work, since some columns are saved as a dict
+ return df
+
+
+def get_valid_filename(s):
+ """
+ Return the given string converted to a string that can be used for a clean
+ filename. Remove leading and trailing spaces; convert other spaces to
+ underscores; and remove anything that is not an alphanumeric, dash,
+ underscore, or dot.
+ >>> get_valid_filename("john's portrait in 2004.jpg")
+ 'johns_portrait_in_2004.jpg'
+ From: https://github.com/django/django/blob/master/django/utils/text.py
+ :param s: input string
+ :return: string of valid filename
+ """
+ s = str(s).strip().replace(' ', '_')
+ return re.sub(r'(?u)[^-\w.]', '', s)
+
+
+def fetch_all_cards_image(df, out_dir=None, size='png'):
+ """
+ Download card images from Scryfall database
+ :param df: pandas dataframe (or series) of cards
+ :param out_dir: path of output directory
+ :param size: Image format given by Scryfall API - 'png', 'large', 'normal', 'small', 'art_crop', 'border_crop'
+ :return:
+ """
+ if size != 'png':
+ print('Note: this repo has been implemented using only \'png\' size. '
+ 'Using %s may result in an unexpected behaviour in other parts of this repo.' % size)
+ if isinstance(df, pd.Series):
+ # df is a single row of card
+ fetch_card_image(df, out_dir, size)
+ else:
+ from concurrent.futures import ThreadPoolExecutor, wait as fwait
+ executor = ThreadPoolExecutor(5)
+ # df is a dataframe containing list of cards
+ arglist = []
+ for ind, row in df.iterrows():
+ arglist.append(executor.submit(fetch_card_image, row, out_dir, size))
+ fwait(arglist)
+ # fetch_card_image(row, out_dir, size)
+
+
+def fetch_card_image(row, out_dir=None, size='png'):
+ """
+ Download a card's image from Scryfall database
+ :param row: pandas series including the card's information
+ :param out_dir: path of the output directory
+ :param size: Image format given by Scryfall API - 'png', 'large', 'normal', 'small', 'art_crop', 'border_crop'
+ :return:
+ """
+ if out_dir is None:
+ s = row['set']
+ if s == 'con':
+ s = 'con__'
+ out_dir = '%s/card_img/%s/%s' % (Config.data_dir, size, s)
+ if not os.path.exists(out_dir):
+ os.makedirs(out_dir)
+
+ # Extract card's name and URL for image accordingly
+ # Double-faced cards have a different format, and results in two separate card images
+ png_urls = []
+ card_names = []
+ if row['layout'] in ['transform', 'double_faced_token']:
+ if isinstance(row['card_faces'], str): # For some reason, dict isn't being parsed in the previous step
+ card_faces = ast.literal_eval(row['card_faces'])
+ else:
+ card_faces = row['card_faces']
+ for i in range(len(card_faces)):
+ png_urls.append(card_faces[i]['image_uris'][size])
+ card_names.append(get_valid_filename(card_faces[i]['name']))
+ else: #if row['layout'] == 'normal':
+ if isinstance(row['image_uris'], str): # For some reason, dict isn't being parsed in the previous step
+ png_urls.append(ast.literal_eval(row['image_uris'])[size])
+ else:
+ png_urls.append(row['image_uris'][size])
+ card_names.append(get_valid_filename(row['name']))
+
+ for i in range(len(png_urls)):
+ img_name = '%s/%s_%s.png' % (out_dir, row['collector_number'], card_names[i])
+ if not os.path.isfile(img_name):
+ request.urlretrieve(png_urls[i], filename=img_name)
+ print(img_name)
+
+
+def main():
+ # Query card data by each set, then merge them together
+ if not os.path.exists(os.path.join(Config.data_dir, 'csv')):
+ os.mkdir(os.path.join(Config.data_dir, 'csv'))
+ for set_name in Config.all_set_list:
+ set_name_m = set_name
+ if set_name_m == 'con':
+ set_name_m = 'con__'
+ csv_name = '%s/csv/%s.csv' % (Config.data_dir, set_name_m)
+ print(csv_name)
+ if not os.path.isfile(csv_name):
+ df = fetch_all_cards_text(url='https://api.scryfall.com/cards/search?q=set:%s+lang:en' % set_name,
+ csv_name=csv_name)
+ else:
+ df = load_all_cards_text(csv_name)
+ df.sort_values('collector_number')
+ fetch_all_cards_image(df, out_dir='%s/card_img/png/%s' % (Config.data_dir, set_name))
+
+ #df = fetch_all_cards_text(url='https://api.scryfall.com/cards/search?q=layout:normal+lang:en+frame:2003',
+ # csv_name='%s/csv/all.csv' % Config.data_dir)
+ return
+
+
+if __name__ == '__main__':
+ main()
--
Gitblit v1.10.0