import httplib2 import re from bs4 import BeautifulSoup if __name__ == '__main__': h = httplib2.Http('.cache') site = 'https://realpornclip.com/wp-content/uploads/2020/06/' response, content = h.request(site) s = BeautifulSoup(content) """find only file names""" links = s.find_all(href=re.compile('\..*$')) for link in links: print(site + link['href'])