Python type hinting – Code Review Stack Exchange

I’m new to using type hinting in Python. I’ve used it for a small scraper I had to build (see code below), everything works fine and mypy gives no errors. However I’m sure there are better ways to write this (avoiding the repetition between the Ratings and ScrapedData tuples, better way to handle the Literal in function signature). Any feedback is greatly appreciated, even on other aspects of the code.

I’m using Python 3.7 so I don’t think I can use TypedDict.

import os
import requests
import lxml.html
import pandas as pd
from lxml.html import HtmlElement
from requests import Session
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
from enum import Enum
from typing import List, Optional, NamedTuple
from typing_extensions import Literal
from multiprocessing import Pool

HEADER = {"User-Agent": "Mozilla/5.0"}

TITLE_XPATH = '//div(@class="review-title")'
REVIEW_XPATH = '//section(@class="review-container")'
SENTIMENT_XPATH = '//div(@class="left-header")'

RATING_XPATH = '//section(@itemprop="reviewrating")'
SUBJECT_XPATH = './/div(@class="subject")'
STAR_XPATH = './/span(@class="or-sprite-inline-block common_yellowstar_desktop")'

POSITIVE_XPATH = './/div(contains(@class, "smiley_smile"))'
NEUTRAL_XPATH = './/div(contains(@class, "smiley_ok"))'
NEGATIVE_XPATH = './/div(contains(@class, "smiley_cry"))'


class Evaluation(Enum):
    POSITIVE: int = 1
    NEUTRAL: int = 0
    NEGATIVE: int = -1
    NONE: None = None


class Ratings(NamedTuple):

    taste: Optional(int) = None
    environment: Optional(int) = None
    service: Optional(int) = None
    hygiene: Optional(int) = None
    value: Optional(int) = None


class ScrapedData(NamedTuple):
    url: str
    title: Optional(str) = None
    review: Optional(str) = None
    sentiment: Literal(
        Evaluation.POSITIVE, Evaluation.NEUTRAL, Evaluation.NEGATIVE, Evaluation.NONE
    ) = Evaluation.NONE
    taste: Optional(int) = None
    environment: Optional(int) = None
    service: Optional(int) = None
    hygiene: Optional(int) = None
    value: Optional(int) = None


class Scraper:
    def __init__(self, url_file: str) -> None:

        if not os.path.exists(url_file):
            raise OSError("File Not Found: %s" % url_file)

        with open(url_file, "r") as fp:
            self.urls = (_.strip() for _ in fp.readlines())

        self.data: list = ()

    @staticmethod
    def __requests_retry_session(
        retries: int = 3,
        backoff_factor: float = 0.3,
        status_forcelist: tuple = (500, 502, 504),
        session: Session = None,
    ) -> Session:
        """
        Handles retries for request HTTP requests params are similar to those
        for requests.packages.urllib3.util.retry.Retry
        https://www.peterbe.com/plog/best-practice-with-retries-with-requests
        """
        session = session or requests.Session()
        retry = Retry(
            total=retries,
            read=retries,
            connect=retries,
            backoff_factor=backoff_factor,
            status_forcelist=status_forcelist,
        )
        adapter = HTTPAdapter(max_retries=retry)
        session.mount("http://", adapter)
        session.mount("https://", adapter)
        return session

    @staticmethod
    def __safe_extract_text(elements: List(HtmlElement)) -> Optional(str):
        """
        Returns the text content of the first element extracted from Xpath or None if none has been found
        :param elements:
        The result of a call to .xpath on the tree
        :return: the string extracted or None if there are no elements
        """
        if len(elements) > 0:
            return elements(0).text_content()
        else:
            return None

    @staticmethod
    def __extract_sentiment(
        elements: List(HtmlElement)
    ) -> Literal(
        Evaluation.POSITIVE, Evaluation.NEUTRAL, Evaluation.NEGATIVE, Evaluation.NONE
    ):

        if len(elements) < 1:
            return Evaluation.NONE
        element = elements(0)
        if len(element.xpath(POSITIVE_XPATH)) > 0:
            return Evaluation.POSITIVE
        elif len(element.xpath(NEUTRAL_XPATH)) > 0:
            return Evaluation.NEUTRAL
        elif len(element.xpath(NEGATIVE_XPATH)) > 0:
            return Evaluation.NEGATIVE
        return Evaluation.NONE

    @staticmethod
    def __extract_ratings(elements) -> Ratings:

        if len(elements) < 1:
            return Ratings()

        element = elements(0)
        rating_subjects = element.xpath(SUBJECT_XPATH)
        if len(rating_subjects) != 5:
            return Ratings()

        extracted_ratings = Ratings(
            *(len(_.xpath(STAR_XPATH)) for _ in rating_subjects)
        )

        return extracted_ratings

    def scrape_page(self, url: str) -> ScrapedData:

        print("Scraping : %s" % url)
        r = self.__requests_retry_session().get(url, headers=HEADER, timeout=10)
        tree = lxml.html.fromstring(r.content)

        # Extract title
        title = self.__safe_extract_text(tree.xpath(TITLE_XPATH))

        # Extract review
        review = self.__safe_extract_text(tree.xpath(REVIEW_XPATH))

        # Extract overall sentiment
        sentiment = self.__extract_sentiment(tree.xpath(SENTIMENT_XPATH))

        # Extract specific grades
        ratings = self.__extract_ratings(tree.xpath(RATING_XPATH))

        return ScrapedData(
            url, title, review, sentiment.value, *ratings._asdict().values()
        )

    def scrape(self) -> None:
        p = Pool(5)
        self.data = p.map(self.scrape_page, self.urls)
        p.terminate()
        p.join()

    def save(self, output_file: str = "content.csv"):
        data = pd.DataFrame(self.data)
        data.to_csv(output_file, index=None)


if __name__ == "__main__":
    s = Scraper("reviewsurl.csv")
    s.scrape()
    s.save()

COMPLEX PYTHON HANGMAN ASAP – Code Review Stack Exchange

i have a project to submit a python program, and not just any python program but a complex one. and with my luck i got hangman. so now i have to make it complex. need help.
I know the basic hangman coding help me add a quiz instead of guessing the word
HERE’S MY CODE

from random import randint
def core_game():

def init_hangman():
    hangman = ()

    for x in range(7):
        hangman.append((" ") * 7)
    hangman(0)(0) = "_"
    hangman(0)(1) = "_"
    hangman(0)(2) = "_"
    hangman(1)(3) = "|"
    return hangman

hangman = init_hangman()

def print_hangman():
    for x in hangman:
        print(str.join("", x))

def get_input(guess):
        your_guess = input(guess)
        if your_guess in guessed_letters:
            print("You already guessed that letter!")
            return get_input(guess)
        elif your_guess.isalpha() and len(your_guess) == 1:
            return your_guess
        else:
            print("Please guess a single letter!")
            return get_input(guess)

words_list = ("monkey", "cow")
city_list = ("Amarillo", "Houston")
lists = ("animals", "cities")
random_list = randint(0,1)
random_word = randint(0,1)
if lists(random_list) == "cities":
    rand_list = city_list
elif lists(random_list) == "animals":
    rand_list = words_list
word = rand_list(random_word)
guessed = ""
guessed_letters = ()
hang = 6
Cont = True

for letter in word:
    guessed += "-"

print("nnnWELCOME TO HANGMAN!")    
print("The category is: ", lists(random_list))
print("The secret word: ", guessed, "is", len(guessed), "letters")

while Cont:
    your_guess = get_input("nEnter your guess: ")
    if your_guess in word.lower():           
        for x in range(len(word)):
            if word(x).lower() == your_guess.lower():
                guessed = guessed(:x) + word(x) + guessed(x+1:)
        guessed_letters.append(your_guess)
        print("nThe secret word: ", guessed)
        if guessed.lower() == word.lower():
            print("nnCongratulations, you guessed the word!")
            play_again = input("nWould you like to play again?(y/n) ")
            if play_again == "y" or play_again == "yes":
                core_game()
            else:
                Cont = False

    else:
        hang -= 1
        guessed_letters.append(your_guess)
        print("nGuessed letters: ", guessed_letters)
        if hang == 5:
            hangman(2)(3) = "O"
            print_hangman()
            print(guessed)
        elif hang == 4:
            hangman(3)(3) = "|"
            print_hangman()
            print(guessed)
        elif hang == 3:
            hangman(3)(2) = "-"
            print_hangman()
            print(guessed)
        elif hang == 2:
            hangman(3)(4) = "-"
            print_hangman()
            print(guessed)
        elif hang == 1:
            hangman(4)(2) = "https://codereview.stackexchange.com/"
            print_hangman()
            print(guessed)
        elif hang == 0:
            hangman(4)(4) = "\"
            print_hangman()
            print("Game Over!")
            print("The word was: ", word)
            play_again = input("Would you like to play again?(y/n) ")
            if play_again == "y" or play_again == "yes":
                core_game()
            else:
                Cont = False

core_game()

✅LEGIT – Earnfreecryptocurency.com Review: SCAM or LEGIT? | NewProxyLists

Earnings Disclaimer:  All the posts published herein are merely based on individual views, and they do not expressly or by implications represent those of NewProxyLists or its owner. It is hereby made clear that NewProxyLists does not endorse, support, adopt or vouch any views, programs and/or business opportunities posted herein. NewProxyLists also does not give and/or offer any investment advice to any members and/or it’s readers. All members and readers are advised to independently consult their own consultants, lawyers and/or families before making any investment and/or business decisions. This forum is merely a place for general discussions. It is hereby agreed by all members and/or readers that NewProxyLists is in no way responsible and/or liable for any damages and/or losses suffered by anyone of you.

Review: myrlse.com | NewProxyLists – Webmaster Forum

Hello! I relaunched my old website (myrls.se) abandoned some years ago, now (myrlse.com)

What can you find on my website?

Games, Movies, TV Shows

I dont upload to any filehost like MEGA or Mediafire just to Google Drive.

My website dont have ADS, users can donate to help if they want.

⚡ MyRLS – Daily Updated Free Downloads

OLD (2017) ⚡ NEW (2020)

What do you think?

✅https://myrlse.com/

Kevincheri – Review | Web Hosting Talk

Thumbs up Kevincheri – Review

Kevin (kevincheri) has been a great professional offering support for my company. We need help managing CentOS servers, he has quickly solved the problems we had.

A few months ago he configured us a machine with CentOS + webmin, an impeccable job. He also configured the security of a CentOS + cPanel machine.
In the last few days he fixed a problem with DirectAdmin + SSL.

I wrote few reviews, but Kewin works very fast and has solved our problems at the right time.

Kevin Cheri
Skype: lynxmaestro

WebSitePanel / MspControl / SolidCP / Smartermail / Installation / Configuration / Troubleshooting / Migrations
Windows Server Management / Security / Hardening
I speak English and Spanish

Review this Image Tool please (sort of online photoshop)

Review this Image Tool please (kinda online photoshop)

Hi,
Kindly review the image design tool on this page:

https://vmapp.org/imageMake/logo.php

I don’t need HTML website design reviews, design is not done yet.
Review the tool please, quality of images its generating, is it easy enough to use etc. If you would like me to add something else etc.

This is not like regular “logo designers” which we come across online.
In those tools, webmasters have already made templates and you can just write your company’s name there.

This one is something more like photoshop, but online.
It can do some of the things that photoshop does, like color a random shape or text, gradient fill it, skew it, distort perspective it, drop different types of shadows, give a brightness shine effect to one side, etc.
Of course its not as advanced as photoshop, nowhere even close I think, but I tried to replicate some functionality using PHP alone, no imagemagic.

The parts I like most are control of gradient percentages, and the CSS type “auto positioning of layers”, and auto deciding suitable width and height of background.
Something like what browsers do with images and CSS float.

For example, while adding layers, instead of giving it a fixed x,y position on the background, you can give it a CSS value like “right”
When you will click “Make Logo” button, processing starts, and it will auto fit “this layer” to the right of “previous layer” processed.
Keywords supported are: “top, bottom, left, right”
SEMrush

The example image which you see at the bottom of the page, that is completely CSS styled, no fixed positions given.
The width and height of the total background was also auto decided by the tool itself, I only specified colors, left the “background width/height” section empty.

Once the image is made, it will be displayed at the bottom of the page (where default image is).
You will also see 2 links “download VMapp file”, and “download png file”. Download them both.

VMapp file is like a PSD file. You can later upload it and make modifications to existing images. No need to remake the whole thing again.
Basically a json set, nothing complicated…

You can move layers up/down much like how you use “align layers” in photoshop. Use the arrows next to a layer’s name.
By the way, that arrow was also made using this tool itself. Minimized a shape from the gallery too much and used the circle setting.

Another thing it supports is “short hand for colors”.
So if you want to have full white color, you need not type full hex “”FFFFFF” or “255.255.255”, simply type “f” or “ff” etc and it will accept it.
You can type just “b5” instead of “b5b5b5”

Try adding one or two text layers, one or two shape layers and click the “make logo” button.

When you make a new layer, click the “Add” button. “Save” button is to save changes to existing layers and not to save new layers.

Everything is keyboard based, excuse me for that. I did that so that even people with very bad eyesight can use it and they need not struggle with mouse…

Please let me know what you think about it.
My main concern is the quality of image that will be generated finally.

Thanks

 

Review: pornklics.com | NewProxyLists – Webmaster Forum

I’ve just started,please don’t be so cruel..i know ..i’ve to work a lot again..any suggestion is welcommed

Thanks

Also hiring uploaders….30 of your movies….for 5 $ ..at my condition:if u feel like check this before ,please:
POSTING RULES

No ”foxy,, people,please

❕NEWS – picoworkers review | NewProxyLists

hello
picoworkers is one of the best microjobs site
the interface is very easy to use and earning are from 0.08 usd to 1.2 usd per task
all microtasks are available;signups,social media,surveys,app download…..
you can earn more than 10 per day
and every instant ,you can find new jobs
minimal withdraw is 10 usd via paypal
picpicoworkers link

Review: Tentrending.com | NewProxyLists – Webmaster Forum

www.Tentrending.com. My newly (somewhat) azm affiliate based site. More on wondering if being with them worth it ever since they slashed their rates (this year) yet again.

But still kinda tempting knowing sells can be made there and the fact that aren’t many that alternatives outside amz.

 

Review my website: Funny pictures, gifs and etc..

Hey guys, can you review my new web site for funny pictures, gifs, jokes and etc..
You can see it here : http://breakingfun.com

Thanks!

DreamProxies - Cheapest USA Elite Private Proxies 100 Private Proxies 200 Private Proxies 400 Private Proxies 1000 Private Proxies 2000 Private Proxies ExtraProxies.com - Buy Cheap Private Proxies Buy 50 Private Proxies Buy 100 Private Proxies Buy 200 Private Proxies Buy 500 Private Proxies Buy 1000 Private Proxies Buy 2000 Private Proxies ProxiesLive Proxies-free.com New Proxy Lists Every Day Proxies123