76 lines
1.2 KiB
Python
76 lines
1.2 KiB
Python
from app.exceptions.invalid_text_exception import InvalidTextException
|
|
from app.moderation.profanity.detector import ProfanityDetector
|
|
from app.ml.text_classifier import TextClassifier
|
|
from app.ml.prediction_result import PredictionResult
|
|
from app.config.settings import settings
|
|
|
|
|
|
class TextModerationService:
|
|
|
|
def __init__(
|
|
self,
|
|
classifier: TextClassifier,
|
|
profanity_detector: ProfanityDetector
|
|
):
|
|
|
|
self.classifier = classifier
|
|
self.profanity_detector = profanity_detector
|
|
|
|
|
|
def moderate(
|
|
self,
|
|
text: str
|
|
) -> PredictionResult:
|
|
|
|
|
|
if text is None or not text.strip():
|
|
|
|
raise InvalidTextException(
|
|
"Text is empty"
|
|
)
|
|
|
|
|
|
detected_words = (
|
|
self.profanity_detector.detect(text)
|
|
)
|
|
|
|
|
|
if detected_words:
|
|
|
|
|
|
return PredictionResult(
|
|
|
|
label="PROFANITY",
|
|
|
|
score=1.0,
|
|
|
|
approved=False,
|
|
|
|
raw_scores=[],
|
|
|
|
reason="PROFANITY",
|
|
|
|
detected_words=detected_words
|
|
|
|
)
|
|
|
|
|
|
|
|
prediction = self.classifier.predict(text)
|
|
|
|
|
|
|
|
prediction.approved = (
|
|
prediction.score <
|
|
settings.TEXT_TOXIC_THRESHOLD
|
|
)
|
|
|
|
|
|
prediction.reason = (
|
|
"OK"
|
|
if prediction.approved
|
|
else "TOXIC"
|
|
)
|
|
|
|
|
|
return prediction |