From 4902b046b8d4bc5e6c2ed36a28c4243f8db5d629 Mon Sep 17 00:00:00 2001 From: Anna Shtirberg Date: Mon, 25 Jan 2021 13:20:19 +0200 Subject: [PATCH 01/14] translation feature stuff --- app/database/models.py | 2 ++ app/routers/profile.py | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/app/database/models.py b/app/database/models.py index a8a8f9a0..48dec167 100644 --- a/app/database/models.py +++ b/app/database/models.py @@ -28,6 +28,7 @@ class User(Base): email = Column(String, unique=True, nullable=False) password = Column(String, nullable=False) full_name = Column(String) + language = Column(String) description = Column(String, default="Happy new user!") avatar = Column(String, default="profile.png") is_active = Column(Boolean, default=False) @@ -46,6 +47,7 @@ class Event(Base): start = Column(DateTime, nullable=False) end = Column(DateTime, nullable=False) content = Column(String) + translation_content = Column(String) location = Column(String) owner = relationship("User") diff --git a/app/routers/profile.py b/app/routers/profile.py index bb856747..43cf5855 100644 --- a/app/routers/profile.py +++ b/app/routers/profile.py @@ -1,9 +1,9 @@ import io +from PIL import Image from fastapi import APIRouter, Depends, File, Request, UploadFile from starlette.responses import RedirectResponse from starlette.status import HTTP_302_FOUND -from PIL import Image from app import config from app.database.database import get_db @@ -26,6 +26,7 @@ def get_placeholder_user(): email='my@email.po', password='1a2s3d4f5g6', full_name='My Name', + language='english', ) @@ -140,6 +141,6 @@ async def process_image(image, user): def get_image_crop_area(width, height): if width > height: delta = (width - height) // 2 - return (delta, 0, width - delta, height) + return delta, 0, width - delta, height delta = (height - width) // 2 - return (0, delta, width, width + delta) + return 0, delta, width, width + delta From 4bb5f834858a9dea0ff51512abd20b271930db9b Mon Sep 17 00:00:00 2001 From: Anna Shtirberg Date: Mon, 25 Jan 2021 13:33:44 +0200 Subject: [PATCH 02/14] Initial translation stuff --- app/internal/fast_content_translation.py | 91 ++++++++++++++++++++++++ app/routers/fast_translation.py | 16 +++++ 2 files changed, 107 insertions(+) create mode 100644 app/internal/fast_content_translation.py create mode 100644 app/routers/fast_translation.py diff --git a/app/internal/fast_content_translation.py b/app/internal/fast_content_translation.py new file mode 100644 index 00000000..8eae29c4 --- /dev/null +++ b/app/internal/fast_content_translation.py @@ -0,0 +1,91 @@ +from typing import List + +from sqlalchemy.exc import SQLAlchemyError +from translate import Translator +from langdetect import detect + +from app.database.database import SessionLocal +from app.database.models import Event, User + +dictionary_of_languages = { + "de": "german", + "ru": "russian", + "en": "english", + "es": "spanish", + "it": "italian", + "fr": "french", + "he": "hebrew" +} + + +def get_language_for_user( + session: SessionLocal, + user_id: int + ) -> str: + try: + user = ( + session.query(User).filter(User.id == user_id).first() + ) + language_for_user = user.language + except SQLAlchemyError: + return "" + else: + return language_for_user + + +def get_events_per_user( + session: SessionLocal, + user_id: int + ) -> List[Event]: + try: + events = ( + session.query(Event).filter(Event.owner_id == user_id).all() + ) + except SQLAlchemyError: + return [] + else: + return events + + +def get_language_content(content: str) -> str: + return detect(content) + + +def find_language_in_dict(content: str) -> bool: + return get_language_content(content) not in dictionary_of_languages.keys() + + +def compare_languages(language: str, content: str) -> bool: + return dictionary_of_languages[get_language_content(content)] == language + + +def translation(language: str, content: str) -> str: + translator = Translator(from_lang=dictionary_of_languages[get_language_content(content)], to_lang=language) + translation_content = translator.translate(content) + return translation_content + + +def add_translation_to_database(session: SessionLocal, event: Event, translation_content: str): + event.translation_content = translation_content + session.add(event) + session.commit() + session.close() # ??? + + +def fast_content_translation(session: SessionLocal, user_id: int) -> bool: + language = get_language_for_user(session, user_id) + if not language: + return False + events = get_events_per_user(session, user_id) + if not events: + return False + for event in events: + if event.content is None or event.translation_content is not None: + return False + content = event.content + if compare_languages(language, content) and find_language_in_dict(content): + return False + else: + translation_content = translation(language, content) + add_translation_to_database(session, event, translation_content) + return True diff --git a/app/routers/fast_translation.py b/app/routers/fast_translation.py new file mode 100644 index 00000000..db091f4c --- /dev/null +++ b/app/routers/fast_translation.py @@ -0,0 +1,16 @@ +from fastapi import APIRouter, Depends, Request +# from fastapi.templating import Jinja2Templates +from sqlalchemy.orm import Session + +from app.database.database import get_db +# from app.dependencies import templates +from app.internal.fast_content_translation import fast_content_translation + +router = APIRouter() + + +@router.post("/translation") +def fast_translation(db: Session = Depends(get_db)): + user_id = 1 + fast_content_translation(session=db, user_id=user_id) + pass From b2c7ad4d96b55a53604081700c9155846b6ba2d6 Mon Sep 17 00:00:00 2001 From: Anna Shtirberg Date: Wed, 27 Jan 2021 14:25:13 +0200 Subject: [PATCH 03/14] cleaned up models --- app/database/models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/database/models.py b/app/database/models.py index 48dec167..ac79d183 100644 --- a/app/database/models.py +++ b/app/database/models.py @@ -47,7 +47,6 @@ class Event(Base): start = Column(DateTime, nullable=False) end = Column(DateTime, nullable=False) content = Column(String) - translation_content = Column(String) location = Column(String) owner = relationship("User") From ce39245de39546eb0fc152dc6647024c58beb37b Mon Sep 17 00:00:00 2001 From: Anna Shtirberg Date: Wed, 27 Jan 2021 14:25:42 +0200 Subject: [PATCH 04/14] added iso to requirements.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index db0deb54..b0a6f384 100644 --- a/requirements.txt +++ b/requirements.txt @@ -50,3 +50,4 @@ watchgod==0.6 websockets==8.1 wsproto==1.0.0 zipp==3.4.0 +iso-639==0.4.5 From 053d02e0e1cc36bc42a6151f9fbf236e447765cb Mon Sep 17 00:00:00 2001 From: Anna Shtirberg Date: Wed, 27 Jan 2021 14:26:12 +0200 Subject: [PATCH 05/14] added language to user registration --- app/routers/user.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/routers/user.py b/app/routers/user.py index 32dd3ca3..3a10c596 100644 --- a/app/routers/user.py +++ b/app/routers/user.py @@ -7,13 +7,14 @@ from app.internal.utils import save -def create_user(username, password, email, session: Session) -> User: +def create_user(username, password, email, language, session: Session) -> User: """Creates and saves a new user.""" user = User( username=username, password=password, email=email, + language=language ) save(user, session=session) return user From 47570a58c078377aafa1824c5de42253f8f50569 Mon Sep 17 00:00:00 2001 From: Anna Shtirberg Date: Wed, 27 Jan 2021 14:27:32 +0200 Subject: [PATCH 06/14] added tests --- tests/test_user.py | 2 ++ tests/user_fixture.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tests/test_user.py b/tests/test_user.py index 9e2a9a84..20c30e87 100644 --- a/tests/test_user.py +++ b/tests/test_user.py @@ -9,10 +9,12 @@ def test_create_user(self, session): username='new_test_username', password='new_test_password', email='new_test.email@gmail.com', + language='english' ) assert user.username == 'new_test_username' assert user.password == 'new_test_password' assert user.email == 'new_test.email@gmail.com' + assert user.language == 'english' session.delete(user) session.commit() diff --git a/tests/user_fixture.py b/tests/user_fixture.py index 526cc10f..f380c0e7 100644 --- a/tests/user_fixture.py +++ b/tests/user_fixture.py @@ -12,6 +12,7 @@ def user(session: Session) -> User: username='test_username', password='test_password', email='test.email@gmail.com', + language='english' ) yield test_user delete_instance(session, test_user) @@ -24,6 +25,7 @@ def sender(session: Session) -> User: username='sender_username', password='sender_password', email='sender.email@gmail.com', + language='english' ) yield sender delete_instance(session, sender) From 08e392ca21e58c1f41fbf272ffaf0d373eaa0b97 Mon Sep 17 00:00:00 2001 From: Anna Shtirberg Date: Wed, 27 Jan 2021 14:28:42 +0200 Subject: [PATCH 07/14] Removed unused files --- app/internal/fast_content_translation.py | 91 ------------------------ app/routers/fast_translation.py | 16 ----- 2 files changed, 107 deletions(-) delete mode 100644 app/internal/fast_content_translation.py delete mode 100644 app/routers/fast_translation.py diff --git a/app/internal/fast_content_translation.py b/app/internal/fast_content_translation.py deleted file mode 100644 index 8eae29c4..00000000 --- a/app/internal/fast_content_translation.py +++ /dev/null @@ -1,91 +0,0 @@ -from typing import List - -from sqlalchemy.exc import SQLAlchemyError -from translate import Translator -from langdetect import detect - -from app.database.database import SessionLocal -from app.database.models import Event, User - -dictionary_of_languages = { - "de": "german", - "ru": "russian", - "en": "english", - "es": "spanish", - "it": "italian", - "fr": "french", - "he": "hebrew" -} - - -def get_language_for_user( - session: SessionLocal, - user_id: int - ) -> str: - try: - user = ( - session.query(User).filter(User.id == user_id).first() - ) - language_for_user = user.language - except SQLAlchemyError: - return "" - else: - return language_for_user - - -def get_events_per_user( - session: SessionLocal, - user_id: int - ) -> List[Event]: - try: - events = ( - session.query(Event).filter(Event.owner_id == user_id).all() - ) - except SQLAlchemyError: - return [] - else: - return events - - -def get_language_content(content: str) -> str: - return detect(content) - - -def find_language_in_dict(content: str) -> bool: - return get_language_content(content) not in dictionary_of_languages.keys() - - -def compare_languages(language: str, content: str) -> bool: - return dictionary_of_languages[get_language_content(content)] == language - - -def translation(language: str, content: str) -> str: - translator = Translator(from_lang=dictionary_of_languages[get_language_content(content)], to_lang=language) - translation_content = translator.translate(content) - return translation_content - - -def add_translation_to_database(session: SessionLocal, event: Event, translation_content: str): - event.translation_content = translation_content - session.add(event) - session.commit() - session.close() # ??? - - -def fast_content_translation(session: SessionLocal, user_id: int) -> bool: - language = get_language_for_user(session, user_id) - if not language: - return False - events = get_events_per_user(session, user_id) - if not events: - return False - for event in events: - if event.content is None or event.translation_content is not None: - return False - content = event.content - if compare_languages(language, content) and find_language_in_dict(content): - return False - else: - translation_content = translation(language, content) - add_translation_to_database(session, event, translation_content) - return True diff --git a/app/routers/fast_translation.py b/app/routers/fast_translation.py deleted file mode 100644 index db091f4c..00000000 --- a/app/routers/fast_translation.py +++ /dev/null @@ -1,16 +0,0 @@ -from fastapi import APIRouter, Depends, Request -# from fastapi.templating import Jinja2Templates -from sqlalchemy.orm import Session - -from app.database.database import get_db -# from app.dependencies import templates -from app.internal.fast_content_translation import fast_content_translation - -router = APIRouter() - - -@router.post("/translation") -def fast_translation(db: Session = Depends(get_db)): - user_id = 1 - fast_content_translation(session=db, user_id=user_id) - pass From 7a725a34515a6897e57ab2dda79e0bc2cd462e04 Mon Sep 17 00:00:00 2001 From: Anna Shtirberg Date: Wed, 27 Jan 2021 14:54:41 +0200 Subject: [PATCH 08/14] added text blob to requirements.txt and made sure it has it's corpora --- app/routers/__init__.py | 3 ++- requirements.txt | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/app/routers/__init__.py b/app/routers/__init__.py index 7170f990..4d1e446a 100644 --- a/app/routers/__init__.py +++ b/app/routers/__init__.py @@ -1,4 +1,5 @@ import nltk - +from textblob import download_corpora nltk.download('punkt') +download_corpora.download_all() diff --git a/requirements.txt b/requirements.txt index 6ab8607a..852e74e2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -87,3 +87,4 @@ word-forms==2.1.0 wsproto==1.0.0 zipp==3.4.0 iso-639==0.4.5 +textblob>=0.15.3 From 4c093987bd6c94cfda2d4fd2292d2d5673a3adc1 Mon Sep 17 00:00:00 2001 From: Anna Shtirberg Date: Wed, 27 Jan 2021 15:01:04 +0200 Subject: [PATCH 09/14] added translation feature and tests --- app/internal/translation.py | 70 ++++++++++++++++++++++ app/routers/__init__.py | 2 - tests/test_translation.py | 116 ++++++++++++++++++++++++++++++++++++ 3 files changed, 186 insertions(+), 2 deletions(-) create mode 100644 app/internal/translation.py create mode 100644 tests/test_translation.py diff --git a/app/internal/translation.py b/app/internal/translation.py new file mode 100644 index 00000000..569addd4 --- /dev/null +++ b/app/internal/translation.py @@ -0,0 +1,70 @@ +from typing import Optional + +from iso639 import languages +from sqlalchemy.exc import SQLAlchemyError +from textblob import TextBlob, download_corpora +from textblob.exceptions import NotTranslated + +from app.database.database import SessionLocal +from app.database.models import User + +download_corpora.download_all() + + +def translate_text(text: str, target_lang: str, original_lang: Optional[str] = None) -> str: + """ + Translate text to the target language + optionally given the original language + """ + if len(text) <= 0: + return "No text to translate" + if original_lang is None: + original_lang = _detect_text_language(text) + else: + original_lang = _lang_full_to_short(original_lang) + + if original_lang == _lang_full_to_short(target_lang): + return text + + try: + return str(TextBlob(text).translate(from_lang=original_lang, to=_lang_full_to_short(target_lang))) + except NotTranslated: + return text + + +def _detect_text_language(text: str) -> str: + """ + Gets some text and returns the language it is in + Uses external API + """ + return str(TextBlob(text).detect_language()) + + +def _get_user_language(user_id: int, session: SessionLocal) -> str: + """ + Gets a user-id and returns the language he speaks + Uses the DB""" + try: + user = ( + session.query(User).filter(User.id == user_id).first() + ) + language_user = user.language + except SQLAlchemyError: + return "" + else: + return language_user + + +def translate_text_for_user(text: str, session: SessionLocal, user_id: int) -> str: + """ + Gets a text and a user-id and returns the text, + translated to the language the user speaks + """ + target_lang = _get_user_language(user_id, session) + if not target_lang: + return text + return translate_text(text, target_lang) + + +def _lang_full_to_short(full_lang: str) -> str: + return languages.get(name=full_lang.capitalize()).alpha2 diff --git a/app/routers/__init__.py b/app/routers/__init__.py index 4d1e446a..7c5175f1 100644 --- a/app/routers/__init__.py +++ b/app/routers/__init__.py @@ -1,5 +1,3 @@ import nltk -from textblob import download_corpora nltk.download('punkt') -download_corpora.download_all() diff --git a/tests/test_translation.py b/tests/test_translation.py new file mode 100644 index 00000000..702bf00d --- /dev/null +++ b/tests/test_translation.py @@ -0,0 +1,116 @@ +import pytest +from iso639 import languages +from textblob import TextBlob + +from app.internal.translation import \ + translate_text, \ + translate_text_for_user, \ + _get_user_language, \ + _lang_full_to_short, \ + _detect_text_language + + +@pytest.mark.parametrize("text, target_lang, original_lang", + [("Привет мой друг", "english", "russian"), + ("Hola mi amigo", "english", "spanish"), + ("Bonjour, mon ami", "english", "french"), + ("Hallo, mein Freund", "english", "german"), + ]) +def test_translate_text_with_original_lang(text, target_lang, original_lang): + answer = translate_text(text, target_lang, original_lang) + assert "Hello my friend" == answer + assert TextBlob(text).detect_language() == languages.get(name=original_lang.capitalize()).alpha2 + assert TextBlob(answer).detect_language() == languages.get(name=target_lang.capitalize()).alpha2 + + +@pytest.mark.parametrize("text, target_lang", + [("Привет мой друг", "english"), + ("Bonjour, mon ami", "english"), + ("Hallo, mein Freund", "english"), + ]) +def test_translate_text_without_original_lang(text, target_lang): + answer = translate_text(text, target_lang) + assert "Hello my friend" == answer + assert TextBlob(answer).detect_language() == languages.get(name=target_lang.capitalize()).alpha2 + + +@pytest.mark.parametrize("text, target_lang, original_lang", + [("Привет мой друг", "russian", "russian"), + ("Hola mi amigo", "spanish", "spanish"), + ("Bonjour, mon ami", "french", "french"), + ("Hallo, mein Freund", "german", "german"), + ("Ciao amico", "italian", "italian") + ]) +def test_translate_text_with_same_original_target_lang_with_original_lang(text, target_lang, original_lang): + answer = translate_text(text, target_lang, original_lang) + assert answer == text + + +@pytest.mark.parametrize("text, target_lang", + [("Привет мой друг", "russian"), + ("Hola mi amigo", "spanish"), + ("Bonjour, mon ami", "french"), + ("Hallo, mein Freund", "german"), + ("Ciao amico", "italian") + ]) +def test_translate_text_with_same_original_target_lang_without_original_lang(text, target_lang): + answer = translate_text(text, target_lang) + assert answer == text + + +def test_translate_text_without_text_with_original_target_lang(): + answer = translate_text("", "english", "russian") + assert answer == "No text to translate" + + +def test_translate_text_without_text_without_original_lang(): + answer = translate_text("", "english") + assert answer == "No text to translate" + + +def test_lang_short_to_full(): + answer = _lang_full_to_short("english") + assert answer == "en" + + +def test_get_user_language(user, session): + user_id = user.id + answer = _get_user_language(user_id, session=session) + assert user_id == 1 + assert answer.lower() == "english" + + +@pytest.mark.parametrize("text", ["Привет мой друг", + "Bonjour, mon ami", + "Hello my friend"] + ) +def test_translate_text_for_user(text, user, session): + user_id = user.id + answer = translate_text_for_user(text, session, user_id) + assert answer == "Hello my friend" + + +def test_detect_text_language(): + answer = _detect_text_language("Hello my friend") + assert answer == "en" + + +@pytest.mark.parametrize("text, target_lang, original_lang", + [("Hoghhflaff", "english", "spanish"), + ("Bdonfdjourr", "english", "french"), + ("Hafdllnnc", "english", "german"), + ]) +def test_translate_text_with_text_impossible_to_translate(text, target_lang, original_lang): + answer = translate_text(text, target_lang, original_lang) + assert answer == text + + +@pytest.mark.parametrize("text, target_lang, original_lang", + [("@Здравствуй#мой$друг!", "english", "russian"), + ("@Hola#mi$amigo!", "english", "spanish"), + ("@Bonjour#mon$ami!", "english", "french"), + ("@Hallo#mein$Freund!", "english", "german"), + ]) +def test_translate_text_with_symbols(text, target_lang, original_lang): + answer = translate_text(text, target_lang, original_lang) + assert "@ Hello # my $ friend!" == answer From b22c6dc3f9b44948f64911a1ce939463b2df7e0a Mon Sep 17 00:00:00 2001 From: Anna Shtirberg Date: Sun, 31 Jan 2021 00:21:22 +0200 Subject: [PATCH 10/14] sorted requirements.txt and removed duplicated --- requirements.txt | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/requirements.txt b/requirements.txt index e7a6b824..feac4751 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,16 @@ +Jinja2==2.11.2 +MarkupSafe==1.1.1 +Pillow==8.1.0 +PyYAML==5.3.1 +SQLAlchemy==1.3.22 aiofiles==0.6.0 aioredis==1.3.1 aiosmtpd==1.2.2 aiosmtplib==1.1.4 apipkg==1.5 arrow==0.17.0 -asyncio==3.4.3 async-timeout==3.0.1 +asyncio==3.4.3 atomicwrites==1.4.0 atpublic==2.1.2 attrs==20.3.0 @@ -19,11 +24,11 @@ coverage==5.3.1 dnspython==2.1.0 email-validator==1.1.2 execnet==1.7.1 +faker==5.6.2 fakeredis==1.4.5 +fastapi-mail==0.3.3.1 fastapi==0.63.0 flake8==3.8.4 -faker==5.6.2 -fastapi-mail==0.3.3.1 frozendict==1.2 h11==0.12.0 h2==4.0.0 @@ -37,17 +42,15 @@ idna==2.10 importlib-metadata==3.3.0 inflect==4.1.0 iniconfig==1.1.1 -Jinja2==2.11.2 -mccabe==0.6.1 +iso-639==0.4.5 joblib==1.0.0 lazy-object-proxy==1.5.2 loguru==0.5.3 -mypy==0.790 +mccabe==0.6.1 mypy-extensions==0.4.3 -MarkupSafe==1.1.1 +mypy==0.790 nltk==3.5 packaging==20.8 -Pillow==8.1.0 pluggy==0.13.1 priority==1.3.0 psycopg2==2.8.6 @@ -56,21 +59,20 @@ pycodestyle==2.6.0 pydantic==1.7.3 pyflakes==2.2.0 pyparsing==2.4.7 -pytest==6.2.1 pytest-asyncio==0.14.0 pytest-cov==2.10.1 -pytest-mock==3.5.1 pytest-forked==1.3.0 +pytest-mock==3.5.1 pytest-xdist==2.2.0 +pytest==6.2.1 python-dateutil==2.8.1 python-dotenv==0.15.0 python-multipart==0.0.5 pytz==2020.5 -PyYAML==5.3.1 redis==3.5.3 regex==2020.11.13 -requests==2.25.1 requests-mock==1.8.0 +requests==2.25.1 responses==0.12.1 rfc3986==1.4.0 six==1.15.0 @@ -78,10 +80,9 @@ smtpdfix==0.2.6 sniffio==1.2.0 sortedcontainers==2.3.0 soupsieve==2.1 -sortedcontainers==2.3.0 -SQLAlchemy==1.3.22 starlette==0.13.6 text-unidecode==1.3 +textblob>=0.15.3 toml==0.10.2 tqdm==4.56.0 typed-ast==1.4.2 @@ -93,5 +94,3 @@ websockets==8.1 word-forms==2.1.0 wsproto==1.0.0 zipp==3.4.0 -iso-639==0.4.5 -textblob>=0.15.3 From 20c5c6050b90bcfe67f3135073d7ad972605648c Mon Sep 17 00:00:00 2001 From: Anna Shtirberg Date: Sun, 31 Jan 2021 00:33:09 +0200 Subject: [PATCH 11/14] Fixed flake8 error --- app/internal/translation.py | 13 ++++++++++--- tests/test_translation.py | 34 +++++++++++++++++++++++----------- 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/app/internal/translation.py b/app/internal/translation.py index 569addd4..ec808bfc 100644 --- a/app/internal/translation.py +++ b/app/internal/translation.py @@ -11,7 +11,10 @@ download_corpora.download_all() -def translate_text(text: str, target_lang: str, original_lang: Optional[str] = None) -> str: +def translate_text(text: str, + target_lang: str, + original_lang: Optional[str] = None + ) -> str: """ Translate text to the target language optionally given the original language @@ -27,7 +30,9 @@ def translate_text(text: str, target_lang: str, original_lang: Optional[str] = N return text try: - return str(TextBlob(text).translate(from_lang=original_lang, to=_lang_full_to_short(target_lang))) + return str(TextBlob(text).translate( + from_lang=original_lang, + to=_lang_full_to_short(target_lang))) except NotTranslated: return text @@ -55,7 +60,9 @@ def _get_user_language(user_id: int, session: SessionLocal) -> str: return language_user -def translate_text_for_user(text: str, session: SessionLocal, user_id: int) -> str: +def translate_text_for_user(text: str, + session: SessionLocal, + user_id: int) -> str: """ Gets a text and a user-id and returns the text, translated to the language the user speaks diff --git a/tests/test_translation.py b/tests/test_translation.py index 702bf00d..35b18025 100644 --- a/tests/test_translation.py +++ b/tests/test_translation.py @@ -2,12 +2,13 @@ from iso639 import languages from textblob import TextBlob -from app.internal.translation import \ - translate_text, \ - translate_text_for_user, \ - _get_user_language, \ - _lang_full_to_short, \ +from app.internal.translation import ( + translate_text, + translate_text_for_user, + _get_user_language, + _lang_full_to_short, _detect_text_language +) @pytest.mark.parametrize("text, target_lang, original_lang", @@ -19,8 +20,10 @@ def test_translate_text_with_original_lang(text, target_lang, original_lang): answer = translate_text(text, target_lang, original_lang) assert "Hello my friend" == answer - assert TextBlob(text).detect_language() == languages.get(name=original_lang.capitalize()).alpha2 - assert TextBlob(answer).detect_language() == languages.get(name=target_lang.capitalize()).alpha2 + assert TextBlob(text).detect_language() == languages.get( + name=original_lang.capitalize()).alpha2 + assert TextBlob(answer).detect_language() == languages.get( + name=target_lang.capitalize()).alpha2 @pytest.mark.parametrize("text, target_lang", @@ -31,7 +34,8 @@ def test_translate_text_with_original_lang(text, target_lang, original_lang): def test_translate_text_without_original_lang(text, target_lang): answer = translate_text(text, target_lang) assert "Hello my friend" == answer - assert TextBlob(answer).detect_language() == languages.get(name=target_lang.capitalize()).alpha2 + assert TextBlob(answer).detect_language() == languages.get( + name=target_lang.capitalize()).alpha2 @pytest.mark.parametrize("text, target_lang, original_lang", @@ -41,7 +45,10 @@ def test_translate_text_without_original_lang(text, target_lang): ("Hallo, mein Freund", "german", "german"), ("Ciao amico", "italian", "italian") ]) -def test_translate_text_with_same_original_target_lang_with_original_lang(text, target_lang, original_lang): +def test_translate_text_with_same_original_target_lang_with_original_lang( + text, + target_lang, + original_lang): answer = translate_text(text, target_lang, original_lang) assert answer == text @@ -53,7 +60,9 @@ def test_translate_text_with_same_original_target_lang_with_original_lang(text, ("Hallo, mein Freund", "german"), ("Ciao amico", "italian") ]) -def test_translate_text_with_same_original_target_lang_without_original_lang(text, target_lang): +def test_translate_text_with_same_original_target_lang_without_original_lang( + text, + target_lang): answer = translate_text(text, target_lang) assert answer == text @@ -100,7 +109,10 @@ def test_detect_text_language(): ("Bdonfdjourr", "english", "french"), ("Hafdllnnc", "english", "german"), ]) -def test_translate_text_with_text_impossible_to_translate(text, target_lang, original_lang): +def test_translate_text_with_text_impossible_to_translate( + text, + target_lang, + original_lang): answer = translate_text(text, target_lang, original_lang) assert answer == text From fb0a76d904afce17f7ba296277347266717dbe51 Mon Sep 17 00:00:00 2001 From: Anna Shtirberg Date: Mon, 1 Feb 2021 14:51:20 +0200 Subject: [PATCH 12/14] Added test to translation and fixed stuff Added logs Fixed problems with pull request Added documentation Added typing information --- app/internal/translation.py | 16 ++++++++++------ app/routers/user.py | 2 +- tests/test_translation.py | 18 ++++++++++++++++-- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/app/internal/translation.py b/app/internal/translation.py index ec808bfc..afaf80fd 100644 --- a/app/internal/translation.py +++ b/app/internal/translation.py @@ -6,7 +6,8 @@ from textblob.exceptions import NotTranslated from app.database.database import SessionLocal -from app.database.models import User +from loguru import logger +from app.routers.user import get_users download_corpora.download_all() @@ -19,8 +20,8 @@ def translate_text(text: str, Translate text to the target language optionally given the original language """ - if len(text) <= 0: - return "No text to translate" + if not text.strip(): + return "" if original_lang is None: original_lang = _detect_text_language(text) else: @@ -50,11 +51,10 @@ def _get_user_language(user_id: int, session: SessionLocal) -> str: Gets a user-id and returns the language he speaks Uses the DB""" try: - user = ( - session.query(User).filter(User.id == user_id).first() - ) + user = get_users(session, id=user_id)[0] language_user = user.language except SQLAlchemyError: + logger.exception("User of user preferred language was not found in the database.") return "" else: return language_user @@ -74,4 +74,8 @@ def translate_text_for_user(text: str, def _lang_full_to_short(full_lang: str) -> str: + """ + Gets the full language name and + converts it to a two-letter language name + """ return languages.get(name=full_lang.capitalize()).alpha2 diff --git a/app/routers/user.py b/app/routers/user.py index 3a10c596..0a4eafa8 100644 --- a/app/routers/user.py +++ b/app/routers/user.py @@ -7,7 +7,7 @@ from app.internal.utils import save -def create_user(username, password, email, language, session: Session) -> User: +def create_user(username: str, password: str, email: str, language: str, session: Session) -> User: """Creates and saves a new user.""" user = User( diff --git a/tests/test_translation.py b/tests/test_translation.py index 35b18025..e0d19faf 100644 --- a/tests/test_translation.py +++ b/tests/test_translation.py @@ -69,12 +69,12 @@ def test_translate_text_with_same_original_target_lang_without_original_lang( def test_translate_text_without_text_with_original_target_lang(): answer = translate_text("", "english", "russian") - assert answer == "No text to translate" + assert answer == "" def test_translate_text_without_text_without_original_lang(): answer = translate_text("", "english") - assert answer == "No text to translate" + assert answer == "" def test_lang_short_to_full(): @@ -126,3 +126,17 @@ def test_translate_text_with_text_impossible_to_translate( def test_translate_text_with_symbols(text, target_lang, original_lang): answer = translate_text(text, target_lang, original_lang) assert "@ Hello # my $ friend!" == answer + + +@pytest.mark.parametrize("text, target_lang, original_lang", + [("Привет мой друг", "italian", "spanish"), + ("Hola mi amigo", "english", "russian"), + ("Bonjour, mon ami", "russian", "german"), + ("Ciao amico", "french", "german") + ]) +def test_translate_text_with_with_incorrect_lang( + text, + target_lang, + original_lang): + answer = translate_text(text, target_lang, original_lang) + assert answer == text From 839adbca1c3675445202e25f21e22840590bb79c Mon Sep 17 00:00:00 2001 From: Anna Shtirberg Date: Mon, 1 Feb 2021 22:14:09 +0200 Subject: [PATCH 13/14] Fixed flake8 error --- app/internal/translation.py | 4 +++- app/routers/user.py | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/app/internal/translation.py b/app/internal/translation.py index afaf80fd..0f8b272a 100644 --- a/app/internal/translation.py +++ b/app/internal/translation.py @@ -54,7 +54,9 @@ def _get_user_language(user_id: int, session: SessionLocal) -> str: user = get_users(session, id=user_id)[0] language_user = user.language except SQLAlchemyError: - logger.exception("User of user preferred language was not found in the database.") + logger.exception( + "User of user preferred language was not found in the database." + ) return "" else: return language_user diff --git a/app/routers/user.py b/app/routers/user.py index 0a4eafa8..649f29ec 100644 --- a/app/routers/user.py +++ b/app/routers/user.py @@ -7,7 +7,11 @@ from app.internal.utils import save -def create_user(username: str, password: str, email: str, language: str, session: Session) -> User: +def create_user(username: str, + password: str, + email: str, + language: str, + session: Session) -> User: """Creates and saves a new user.""" user = User( From 797ec8327c2b11cdcbb65b16a6b5df03fe266af0 Mon Sep 17 00:00:00 2001 From: Anna Shtirberg Date: Mon, 1 Feb 2021 22:48:00 +0200 Subject: [PATCH 14/14] add tests, changes in function get_user_language --- app/internal/translation.py | 9 ++++----- tests/test_translation.py | 14 +++++++++++++- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/app/internal/translation.py b/app/internal/translation.py index 0f8b272a..6edef6ef 100644 --- a/app/internal/translation.py +++ b/app/internal/translation.py @@ -1,12 +1,12 @@ from typing import Optional from iso639 import languages -from sqlalchemy.exc import SQLAlchemyError from textblob import TextBlob, download_corpora from textblob.exceptions import NotTranslated from app.database.database import SessionLocal from loguru import logger + from app.routers.user import get_users download_corpora.download_all() @@ -52,14 +52,13 @@ def _get_user_language(user_id: int, session: SessionLocal) -> str: Uses the DB""" try: user = get_users(session, id=user_id)[0] - language_user = user.language - except SQLAlchemyError: + except IndexError: logger.exception( - "User of user preferred language was not found in the database." + "User was not found in the database." ) return "" else: - return language_user + return user.language def translate_text_for_user(text: str, diff --git a/tests/test_translation.py b/tests/test_translation.py index e0d19faf..34c72dcb 100644 --- a/tests/test_translation.py +++ b/tests/test_translation.py @@ -93,12 +93,18 @@ def test_get_user_language(user, session): "Bonjour, mon ami", "Hello my friend"] ) -def test_translate_text_for_user(text, user, session): +def test_translate_text_for_good_user(text, user, session): user_id = user.id answer = translate_text_for_user(text, session, user_id) assert answer == "Hello my friend" +def test_translate_text_for_bed_user(user, session): + user_id = user.id + answer = translate_text_for_user("Привет мой друг", session, user_id + 1) + assert answer == "Привет мой друг" + + def test_detect_text_language(): answer = _detect_text_language("Hello my friend") assert answer == "en" @@ -140,3 +146,9 @@ def test_translate_text_with_with_incorrect_lang( original_lang): answer = translate_text(text, target_lang, original_lang) assert answer == text + + +def test_get_user_language_for_bed_user(user, session): + user_id = user.id + 1 + answer = _get_user_language(user_id, session=session) + assert not answer