stanis-tits-scrap/stanis-tits.py

#!/usr/bin/env python3

import re
import shutil
import os.path
import requests
from bs4 import BeautifulSoup

__author__ = "Alexander Popov"
__version__ = "1.0.1"
__license__ = "Unlicense"

DOWNLOAD_DIR = "./images"


def checkResumeFile():
    if not os.path.exists(
        "{0}/.resume".format(
            DOWNLOAD_DIR,
        )
    ):
        if not os.path.exists(DOWNLOAD_DIR):
            os.mkdir(DOWNLOAD_DIR)

        with open(
            "{0}/.resume".format(
                DOWNLOAD_DIR,
            ),
            "w",
        ) as f:
            f.write("0")
            return [0]
    else:
        with open(
            "{0}/.resume".format(
                DOWNLOAD_DIR,
            ),
            "r",
        ) as f:
            lines = [line.split("\n")[0] for line in f][-20:]

            return lines


def saveResume(resumeList):
    resumeList.sort()
    with open(
        "{0}/.resume".format(
            DOWNLOAD_DIR,
        ),
        "w",
        encoding="utf-8",
    ) as f:
        for item in resumeList[-20:]:
            f.write("{0}\n".format(item))


def getImagesLinks(page):
    URL = lambda page: "http://blog.stanis.ru/?back={0}".format(
        page,
    )
    COOKIES = dict(block="951")

    r = requests.get(URL(page), cookies=COOKIES)
    soup = BeautifulSoup(
        r.text.encode("cp1251"), "html.parser", from_encoding="windows-1251"
    )

    imagesData = soup.findAll("img", src=re.compile("img/*"))

    imagesUrl = list()

    for image in imagesData:
        imagesUrl.append(image["src"].split("/")[1])

    return imagesUrl


def imageDownload(image):
    response = requests.get(
        "https://blog.stanis.ru/imgs/{0}".format(
            image,
        ),
        stream=True,
    )

    with open("{0}/{1}".format(DOWNLOAD_DIR, image), "wb") as out_image:
        shutil.copyfileobj(
            response.raw,
            out_image,
        )


if __name__ == "__main__":
    resumeFiles = checkResumeFile()

    LOOP = True
    downloadPage = 0

    while LOOP:
        imagesLinks = getImagesLinks(downloadPage)
        imagesLinks.sort()

        for image in imagesLinks:
            if not image.split(".")[0] in resumeFiles:
                imageDownload(image)
                resumeFiles.insert(
                    0,
                    image.split(".")[0],
                )
            else:
                LOOP = False

        downloadPage += 1

    saveResume(resumeFiles)