(Python) 픽사베이 이미지 크롤링

1. 이미지 수집하기

• 픽사베이

 

In [1]:

import chromedriver_autoinstaller
import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from urllib.request import Request, urlopen

 

In [3]:

driver = webdriver.Chrome()
driver.implicitly_wait(3)

url = 'https://pixabay.com/ko/images/search/바다/'
driver.get(url)
time.sleep(3)

 

In [5]:

image_xpath = '/html/body/div[1]/div[1]/div/div[2]/div[3]/div/div/div[4]/div[1]/div/a/img'
image_url = driver.find_element(By.XPATH, image_xpath).get_attribute('src')
print('image_url: ', image_url)

 

 

image_url:  https://cdn.pixabay.com/photo/2016/11/29/04/19/ocean-1867285_640.jpg

 

In [9]:

image_byte = Request(image_url, headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36'})
f= open('sea.jpg', 'wb')
f.write(urlopen(image_byte).read())
f.close()

 

 

2. 여러개 이미지 수집하기

 

In [38]:

driver = webdriver.Chrome()
driver.implicitly_wait(3)

url = 'https://pixabay.com/ko/images/search/바다/'
driver.get(url)
time.sleep(3)

image_area_xpath = '/html/body/div[1]/div[1]/div/div[2]/div[3]'
image_area = driver.find_element(By.XPATH, image_area_xpath)
image_elements = image_area.find_elements(By.TAG_NAME, 'img')

image_urls = []

for image_element in image_elements:
    if image_element.get_attribute('data-lazy-src') is None:
        image_url = image_element.get_attribute('src')
    else:
        image_url = image_element.get_attribute('data-lazy-src')
    print(image_url)
    image_urls.append(image_url)

 

 

https://cdn.pixabay.com/photo/2015/04/23/22/00/tree-736885_1280.jpg
https://cdn.pixabay.com/photo/2016/10/18/21/22/beach-1751455_640.jpg
https://cdn.pixabay.com/photo/2015/10/30/20/13/sunrise-1014712_640.jpg
https://cdn.pixabay.com/photo/2014/08/15/11/29/beach-418742_640.jpg
https://cdn.pixabay.com/photo/2016/10/21/14/50/plouzane-1758197_640.jpg
https://cdn.pixabay.com/photo/2016/12/17/14/33/wave-1913559_640.jpg
https://cdn.pixabay.com/photo/2017/05/31/18/38/sea-2361247_640.jpg
https://cdn.pixabay.com/photo/2014/07/16/05/18/beach-394503_640.jpg
https://cdn.pixabay.com/photo/2017/09/16/16/09/sea-2755908_640.jpg
https://cdn.pixabay.com/photo/2016/11/14/22/18/beach-1824855_640.jpg
https://cdn.pixabay.com/photo/2017/04/06/17/43/water-2208931_640.jpg
https://cdn.pixabay.com/photo/2017/02/22/17/02/beach-2089936_640.jpg
https://cdn.pixabay.com/photo/2015/03/05/01/46/sailing-ship-659758_640.jpg
https://cdn.pixabay.com/photo/2018/03/10/17/16/woman-3214594_640.jpg
https://cdn.pixabay.com/photo/2017/02/09/15/10/sea-2052650_640.jpg
https://cdn.pixabay.com/photo/2016/11/06/05/38/boats-1802340_640.jpg
https://cdn.pixabay.com/photo/2016/05/07/16/52/sea-1377712_640.jpg
https://cdn.pixabay.com/photo/2016/11/14/05/29/girl-1822702_640.jpg
https://cdn.pixabay.com/photo/2019/06/20/09/26/underwater-4286600_640.jpg
https://cdn.pixabay.com/photo/2020/03/21/19/27/sea-4955005_640.jpg
https://cdn.pixabay.com/photo/2014/10/07/13/48/mountain-477832_640.jpg
https://cdn.pixabay.com/photo/2018/06/03/14/17/sea-3450584_640.jpg
https://cdn.pixabay.com/photo/2016/10/22/18/52/beach-1761410_640.jpg
https://cdn.pixabay.com/photo/2016/09/07/15/48/sunset-1651878_640.jpg
https://cdn.pixabay.com/photo/2019/01/07/09/43/sunset-3918693_640.jpg
https://cdn.pixabay.com/photo/2013/07/18/20/26/sea-164989_640.jpg
https://cdn.pixabay.com/photo/2018/08/14/13/23/ocean-3605547_640.jpg
https://cdn.pixabay.com/photo/2016/04/18/22/05/seashells-1337565_640.jpg
https://cdn.pixabay.com/photo/2014/08/01/00/08/pier-407252_640.jpg
https://cdn.pixabay.com/photo/2015/03/16/10/59/sunset-675847_640.jpg
https://cdn.pixabay.com/photo/2016/04/15/04/02/water-1330252_640.jpg
https://cdn.pixabay.com/photo/2015/03/30/12/37/jellyfish-698521_640.jpg
https://cdn.pixabay.com/photo/2014/04/26/04/25/woman-332278_640.jpg
https://cdn.pixabay.com/photo/2016/08/17/01/39/mystery-1599527_640.jpg
https://cdn.pixabay.com/photo/2013/10/02/23/03/mountains-190055_640.jpg
https://cdn.pixabay.com/photo/2015/09/09/16/40/girl-931776_640.jpg
https://cdn.pixabay.com/photo/2017/09/18/15/38/moon-2762111_640.jpg
https://cdn.pixabay.com/photo/2017/03/31/15/34/sunset-2191645_640.jpg
https://cdn.pixabay.com/photo/2015/12/27/21/11/beach-1110498_640.jpg
https://cdn.pixabay.com/photo/2018/03/02/21/49/waves-3194377_640.jpg
https://cdn.pixabay.com/photo/2015/03/02/23/40/father-656734_640.jpg
https://cdn.pixabay.com/photo/2016/11/22/23/12/beach-1851101_640.jpg
https://cdn.pixabay.com/photo/2016/05/04/19/39/stones-1372677_640.jpg
https://cdn.pixabay.com/photo/2016/01/23/23/54/mountains-1158269_640.jpg
https://cdn.pixabay.com/photo/2016/02/22/12/37/waves-1215449_640.jpg
https://cdn.pixabay.com/photo/2017/01/16/19/54/sunset-1985086_640.jpg
https://cdn.pixabay.com/photo/2012/03/03/23/52/sea-21649_640.jpg
https://cdn.pixabay.com/photo/2017/02/22/17/06/wave-2089959_640.jpg
https://cdn.pixabay.com/photo/2016/08/14/18/27/sailing-boat-1593613_640.jpg
https://cdn.pixabay.com/photo/2017/06/04/23/17/lighthouse-2372461_640.jpg
https://cdn.pixabay.com/photo/2014/12/16/22/25/woman-570883_640.jpg
https://cdn.pixabay.com/photo/2016/11/23/13/48/beach-1852945_640.jpg
https://cdn.pixabay.com/photo/2018/06/13/18/20/waves-3473335_640.jpg
https://cdn.pixabay.com/photo/2013/12/17/20/10/bubbles-230014_640.jpg
https://cdn.pixabay.com/photo/2017/03/27/14/49/beach-2179183_640.jpg
https://cdn.pixabay.com/photo/2010/12/13/10/09/nature-2384_640.jpg
https://cdn.pixabay.com/photo/2017/03/27/16/50/beach-2179624_640.jpg
https://cdn.pixabay.com/photo/2016/09/07/11/37/sunset-1651426_640.jpg
https://cdn.pixabay.com/photo/2015/07/05/13/44/beach-832346_640.jpg
https://cdn.pixabay.com/photo/2018/10/05/14/39/sunset-3726030_640.jpg
https://cdn.pixabay.com/photo/2018/09/03/23/56/sea-3652697_640.jpg
https://cdn.pixabay.com/photo/2017/04/07/21/22/waves-2211925_640.jpg
https://cdn.pixabay.com/photo/2019/04/27/23/09/beach-4161593_640.jpg
https://cdn.pixabay.com/photo/2017/06/17/18/35/beach-2413081_640.jpg
https://cdn.pixabay.com/photo/2014/07/30/02/00/iceberg-404966_640.jpg
https://cdn.pixabay.com/photo/2014/06/27/12/36/fish-378286_640.jpg
https://cdn.pixabay.com/photo/2014/11/21/03/27/sea-540123_640.jpg
https://cdn.pixabay.com/photo/2014/12/08/17/52/horse-561221_640.jpg
https://cdn.pixabay.com/photo/2013/07/27/05/13/lighthouse-168132_640.jpg
https://cdn.pixabay.com/photo/2018/04/29/21/26/paper-boat-3361172_640.jpg
https://cdn.pixabay.com/photo/2014/08/12/00/01/santorini-416136_640.jpg
https://cdn.pixabay.com/photo/2014/07/05/00/40/waves-384385_640.jpg
https://cdn.pixabay.com/photo/2018/01/16/10/29/waves-3085701_640.jpg
https://cdn.pixabay.com/photo/2023/05/14/19/42/sky-7993656_640.jpg
https://cdn.pixabay.com/photo/2016/11/21/17/09/animals-1846546_640.jpg
https://cdn.pixabay.com/photo/2016/11/29/04/19/ocean-1867285_640.jpg
https://cdn.pixabay.com/photo/2016/03/04/19/36/beach-1236581_640.jpg
https://cdn.pixabay.com/photo/2017/01/20/00/30/maldives-1993704_640.jpg
https://cdn.pixabay.com/photo/2017/12/15/13/51/polynesia-3021072_640.jpg
https://cdn.pixabay.com/photo/2013/11/01/11/13/dolphin-203875_640.jpg
https://cdn.pixabay.com/photo/2016/05/02/10/13/ship-1366926_640.jpg
https://cdn.pixabay.com/photo/2015/03/22/23/07/bora-bora-685303_640.jpg
https://cdn.pixabay.com/photo/2017/10/10/07/48/beach-2836300_640.jpg
https://cdn.pixabay.com/photo/2018/09/19/23/03/sunset-3689760_640.jpg
https://cdn.pixabay.com/photo/2017/08/01/01/17/beach-2562563_640.jpg
https://cdn.pixabay.com/photo/2017/01/28/11/00/white-tailed-eagle-2015098_640.jpg
https://cdn.pixabay.com/photo/2018/01/22/14/13/animal-3099035_640.jpg
https://cdn.pixabay.com/photo/2013/02/21/19/06/drink-84533_640.jpg
https://cdn.pixabay.com/photo/2015/10/30/20/13/boat-1014711_640.jpg
https://cdn.pixabay.com/photo/2018/07/16/16/08/island-3542290_640.jpg
https://cdn.pixabay.com/photo/2013/02/21/19/10/sea-84629_640.jpg
https://cdn.pixabay.com/photo/2016/11/22/19/33/sea-1850228_640.jpg
https://cdn.pixabay.com/photo/2014/10/04/15/07/sunset-473604_640.jpg
https://cdn.pixabay.com/photo/2019/08/06/02/16/mountains-4387209_640.jpg
https://cdn.pixabay.com/photo/2014/06/21/14/25/water-373780_640.jpg
https://cdn.pixabay.com/photo/2017/05/21/15/14/balloons-2331488_640.jpg
https://cdn.pixabay.com/photo/2015/08/30/11/16/sunset-914148_640.jpg
https://cdn.pixabay.com/photo/2014/02/12/16/34/sky-264778_640.jpg
https://cdn.pixabay.com/photo/2016/11/21/15/43/beach-1846040_640.jpg
https://cdn.pixabay.com/photo/2017/12/18/01/41/sea-3025268_640.jpg

 

In [39]:

image_urls

 

Out[39]:

['https://cdn.pixabay.com/photo/2015/04/23/22/00/tree-736885_1280.jpg',
 'https://cdn.pixabay.com/photo/2016/10/18/21/22/beach-1751455_640.jpg',
 'https://cdn.pixabay.com/photo/2015/10/30/20/13/sunrise-1014712_640.jpg',
 'https://cdn.pixabay.com/photo/2014/08/15/11/29/beach-418742_640.jpg',
 'https://cdn.pixabay.com/photo/2016/10/21/14/50/plouzane-1758197_640.jpg',
 'https://cdn.pixabay.com/photo/2016/12/17/14/33/wave-1913559_640.jpg',
 'https://cdn.pixabay.com/photo/2017/05/31/18/38/sea-2361247_640.jpg',
 'https://cdn.pixabay.com/photo/2014/07/16/05/18/beach-394503_640.jpg',
 'https://cdn.pixabay.com/photo/2017/09/16/16/09/sea-2755908_640.jpg',
 'https://cdn.pixabay.com/photo/2016/11/14/22/18/beach-1824855_640.jpg',
 'https://cdn.pixabay.com/photo/2017/04/06/17/43/water-2208931_640.jpg',
 'https://cdn.pixabay.com/photo/2017/02/22/17/02/beach-2089936_640.jpg',
 'https://cdn.pixabay.com/photo/2015/03/05/01/46/sailing-ship-659758_640.jpg',
 'https://cdn.pixabay.com/photo/2018/03/10/17/16/woman-3214594_640.jpg',
 'https://cdn.pixabay.com/photo/2017/02/09/15/10/sea-2052650_640.jpg',
 'https://cdn.pixabay.com/photo/2016/11/06/05/38/boats-1802340_640.jpg',
 'https://cdn.pixabay.com/photo/2016/05/07/16/52/sea-1377712_640.jpg',
 'https://cdn.pixabay.com/photo/2016/11/14/05/29/girl-1822702_640.jpg',
 'https://cdn.pixabay.com/photo/2019/06/20/09/26/underwater-4286600_640.jpg',
 'https://cdn.pixabay.com/photo/2020/03/21/19/27/sea-4955005_640.jpg',
 'https://cdn.pixabay.com/photo/2014/10/07/13/48/mountain-477832_640.jpg',
 'https://cdn.pixabay.com/photo/2018/06/03/14/17/sea-3450584_640.jpg',
 'https://cdn.pixabay.com/photo/2016/10/22/18/52/beach-1761410_640.jpg',
 'https://cdn.pixabay.com/photo/2016/09/07/15/48/sunset-1651878_640.jpg',
 'https://cdn.pixabay.com/photo/2019/01/07/09/43/sunset-3918693_640.jpg',
 'https://cdn.pixabay.com/photo/2013/07/18/20/26/sea-164989_640.jpg',
 'https://cdn.pixabay.com/photo/2018/08/14/13/23/ocean-3605547_640.jpg',
 'https://cdn.pixabay.com/photo/2016/04/18/22/05/seashells-1337565_640.jpg',
 'https://cdn.pixabay.com/photo/2014/08/01/00/08/pier-407252_640.jpg',
 'https://cdn.pixabay.com/photo/2015/03/16/10/59/sunset-675847_640.jpg',
 'https://cdn.pixabay.com/photo/2016/04/15/04/02/water-1330252_640.jpg',
 'https://cdn.pixabay.com/photo/2015/03/30/12/37/jellyfish-698521_640.jpg',
 'https://cdn.pixabay.com/photo/2014/04/26/04/25/woman-332278_640.jpg',
 'https://cdn.pixabay.com/photo/2016/08/17/01/39/mystery-1599527_640.jpg',
 'https://cdn.pixabay.com/photo/2013/10/02/23/03/mountains-190055_640.jpg',
 'https://cdn.pixabay.com/photo/2015/09/09/16/40/girl-931776_640.jpg',
 'https://cdn.pixabay.com/photo/2017/09/18/15/38/moon-2762111_640.jpg',
 'https://cdn.pixabay.com/photo/2017/03/31/15/34/sunset-2191645_640.jpg',
 'https://cdn.pixabay.com/photo/2015/12/27/21/11/beach-1110498_640.jpg',
 'https://cdn.pixabay.com/photo/2018/03/02/21/49/waves-3194377_640.jpg',
 'https://cdn.pixabay.com/photo/2015/03/02/23/40/father-656734_640.jpg',
 'https://cdn.pixabay.com/photo/2016/11/22/23/12/beach-1851101_640.jpg',
 'https://cdn.pixabay.com/photo/2016/05/04/19/39/stones-1372677_640.jpg',
 'https://cdn.pixabay.com/photo/2016/01/23/23/54/mountains-1158269_640.jpg',
 'https://cdn.pixabay.com/photo/2016/02/22/12/37/waves-1215449_640.jpg',
 'https://cdn.pixabay.com/photo/2017/01/16/19/54/sunset-1985086_640.jpg',
 'https://cdn.pixabay.com/photo/2012/03/03/23/52/sea-21649_640.jpg',
 'https://cdn.pixabay.com/photo/2017/02/22/17/06/wave-2089959_640.jpg',
 'https://cdn.pixabay.com/photo/2016/08/14/18/27/sailing-boat-1593613_640.jpg',
 'https://cdn.pixabay.com/photo/2017/06/04/23/17/lighthouse-2372461_640.jpg',
 'https://cdn.pixabay.com/photo/2014/12/16/22/25/woman-570883_640.jpg',
 'https://cdn.pixabay.com/photo/2016/11/23/13/48/beach-1852945_640.jpg',
 'https://cdn.pixabay.com/photo/2018/06/13/18/20/waves-3473335_640.jpg',
 'https://cdn.pixabay.com/photo/2013/12/17/20/10/bubbles-230014_640.jpg',
 'https://cdn.pixabay.com/photo/2017/03/27/14/49/beach-2179183_640.jpg',
 'https://cdn.pixabay.com/photo/2010/12/13/10/09/nature-2384_640.jpg',
 'https://cdn.pixabay.com/photo/2017/03/27/16/50/beach-2179624_640.jpg',
 'https://cdn.pixabay.com/photo/2016/09/07/11/37/sunset-1651426_640.jpg',
 'https://cdn.pixabay.com/photo/2015/07/05/13/44/beach-832346_640.jpg',
 'https://cdn.pixabay.com/photo/2018/10/05/14/39/sunset-3726030_640.jpg',
 'https://cdn.pixabay.com/photo/2018/09/03/23/56/sea-3652697_640.jpg',
 'https://cdn.pixabay.com/photo/2017/04/07/21/22/waves-2211925_640.jpg',
 'https://cdn.pixabay.com/photo/2019/04/27/23/09/beach-4161593_640.jpg',
 'https://cdn.pixabay.com/photo/2017/06/17/18/35/beach-2413081_640.jpg',
 'https://cdn.pixabay.com/photo/2014/07/30/02/00/iceberg-404966_640.jpg',
 'https://cdn.pixabay.com/photo/2014/06/27/12/36/fish-378286_640.jpg',
 'https://cdn.pixabay.com/photo/2014/11/21/03/27/sea-540123_640.jpg',
 'https://cdn.pixabay.com/photo/2014/12/08/17/52/horse-561221_640.jpg',
 'https://cdn.pixabay.com/photo/2013/07/27/05/13/lighthouse-168132_640.jpg',
 'https://cdn.pixabay.com/photo/2018/04/29/21/26/paper-boat-3361172_640.jpg',
 'https://cdn.pixabay.com/photo/2014/08/12/00/01/santorini-416136_640.jpg',
 'https://cdn.pixabay.com/photo/2014/07/05/00/40/waves-384385_640.jpg',
 'https://cdn.pixabay.com/photo/2018/01/16/10/29/waves-3085701_640.jpg',
 'https://cdn.pixabay.com/photo/2023/05/14/19/42/sky-7993656_640.jpg',
 'https://cdn.pixabay.com/photo/2016/11/21/17/09/animals-1846546_640.jpg',
 'https://cdn.pixabay.com/photo/2016/11/29/04/19/ocean-1867285_640.jpg',
 'https://cdn.pixabay.com/photo/2016/03/04/19/36/beach-1236581_640.jpg',
 'https://cdn.pixabay.com/photo/2017/01/20/00/30/maldives-1993704_640.jpg',
 'https://cdn.pixabay.com/photo/2017/12/15/13/51/polynesia-3021072_640.jpg',
 'https://cdn.pixabay.com/photo/2013/11/01/11/13/dolphin-203875_640.jpg',
 'https://cdn.pixabay.com/photo/2016/05/02/10/13/ship-1366926_640.jpg',
 'https://cdn.pixabay.com/photo/2015/03/22/23/07/bora-bora-685303_640.jpg',
 'https://cdn.pixabay.com/photo/2017/10/10/07/48/beach-2836300_640.jpg',
 'https://cdn.pixabay.com/photo/2018/09/19/23/03/sunset-3689760_640.jpg',
 'https://cdn.pixabay.com/photo/2017/08/01/01/17/beach-2562563_640.jpg',
 'https://cdn.pixabay.com/photo/2017/01/28/11/00/white-tailed-eagle-2015098_640.jpg',
 'https://cdn.pixabay.com/photo/2018/01/22/14/13/animal-3099035_640.jpg',
 'https://cdn.pixabay.com/photo/2013/02/21/19/06/drink-84533_640.jpg',
 'https://cdn.pixabay.com/photo/2015/10/30/20/13/boat-1014711_640.jpg',
 'https://cdn.pixabay.com/photo/2018/07/16/16/08/island-3542290_640.jpg',
 'https://cdn.pixabay.com/photo/2013/02/21/19/10/sea-84629_640.jpg',
 'https://cdn.pixabay.com/photo/2016/11/22/19/33/sea-1850228_640.jpg',
 'https://cdn.pixabay.com/photo/2014/10/04/15/07/sunset-473604_640.jpg',
 'https://cdn.pixabay.com/photo/2019/08/06/02/16/mountains-4387209_640.jpg',
 'https://cdn.pixabay.com/photo/2014/06/21/14/25/water-373780_640.jpg',
 'https://cdn.pixabay.com/photo/2017/05/21/15/14/balloons-2331488_640.jpg',
 'https://cdn.pixabay.com/photo/2015/08/30/11/16/sunset-914148_640.jpg',
 'https://cdn.pixabay.com/photo/2014/02/12/16/34/sky-264778_640.jpg',
 'https://cdn.pixabay.com/photo/2016/11/21/15/43/beach-1846040_640.jpg',
 'https://cdn.pixabay.com/photo/2017/12/18/01/41/sea-3025268_640.jpg']

 

In [40]:

for i in range(len(image_urls)):
    image_url = image_urls[i]
    image_byte = Request(image_url, headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36'})
    f= open(f'sea{i}.jpg', 'wb')
    f.write(urlopen(image_byte).read())
    f.close()

 

 

3. 함수로 리팩토링

• crawl_image(keyword, pages)

 

In [47]:

import os
import pathlib

 

In [50]:

def crawl_image(keyword, pages):
    image_urls = []
    for i in range(1, pages+1):
        url = f'https://pixabay.com/ko/images/search/{keyword}/?pagi={i}'
        driver.get(url)
        time.sleep(3)
        image_area_xpath = '/html/body/div[1]/div[1]/div/div[2]/div[3]/div/div'
        image_area = driver.find_element(By.XPATH, image_area_xpath)
        image_elements = image_area.find_elements(By.TAG_NAME, 'img')
        for image_element in image_elements:
            image_url = image_element.get_attribute('data-lazy-src')
            if image_element.get_attribute('data-lazy-src') is None:
                image_url = image_element.get_attribute('src')
            image_urls.append(image_url)
    return image_urls

 

In [56]:

def crawl_and_save_image(keyword, pages):
    image_urls = crawl_image(keyword, pages)
    
    if not os.path.exists(keyword):
        os.mkdir(keyword)
        
    for i in range(len(image_urls)):
        image_url = image_urls[i]
        image_byte = Request(image_url, headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36'})
        filename = image_url.split('/')[-1]
        f= open(f'./{keyword}/{filename}.jpg', 'wb')
        f.write(urlopen(image_byte).read())
        f.close()

 

In [57]:

driver = webdriver.Chrome()
driver.implicitly_wait(3)
crawl_and_save_image('산', 2)