added autocreating csv folder

Constantin Wenger

2022-02-01 be11e02f1d08beb40fff7d54d1c7a343ba10b275

 fetch_data.py

@@ -81,9 +81,14 @@
        # df is a single row of card
        fetch_card_image(df, out_dir, size)
    else:
        from concurrent.futures import ThreadPoolExecutor, wait as fwait
        executor = ThreadPoolExecutor(5)
        # df is a dataframe containing list of cards
        arglist = []
        for ind, row in df.iterrows():
            fetch_card_image(row, out_dir, size)
            arglist.append(executor.submit(fetch_card_image, row, out_dir, size))
        fwait(arglist)
        #    fetch_card_image(row, out_dir, size)


def fetch_card_image(row, out_dir=None, size='png'):
@@ -95,7 +100,10 @@
    :return:
    """
    if out_dir is None:
        out_dir = '%s/card_img/%s/%s' % (Config.data_dir, size, row['set'])
        s = row['set']
        if s == 'con':
           s = 'con__'
        out_dir = '%s/card_img/%s/%s' % (Config.data_dir, size, s)
    if not os.path.exists(out_dir):
        os.makedirs(out_dir)

@@ -127,8 +135,13 @@

def main():
    # Query card data by each set, then merge them together
    if not os.path.exists(os.path.join(Config.data_dir, 'csv')):
        os.mkdir(os.path.join(Config.data_dir, 'csv'))
    for set_name in Config.all_set_list:
        csv_name = '%s/csv/%s.csv' % (Config.data_dir, set_name)
        set_name_m = set_name
        if set_name_m == 'con':
            set_name_m = 'con__'
        csv_name = '%s/csv/%s.csv' % (Config.data_dir, set_name_m)
        print(csv_name)
        if not os.path.isfile(csv_name):
            df = fetch_all_cards_text(url='https://api.scryfall.com/cards/search?q=set:%s+lang:en' % set_name,