added image ai-moderation

ai-moderation/app/services/image_moderation_service.py

@@ -0,0 +1,115 @@
+from PIL import Image
+
+from app.ml.image.image_classifier import ImageClassifier
+from app.ml.image.image_prediction_result import ImagePredictionResult
+from app.ml.image.clip_classifier import ClipClassifier
+from app.moderation.image.validator import ImageValidator
+from app.config.settings import settings
+from app.config.image_policy import FORBIDDEN_IMAGE_LABELS
+
+
+class ImageModerationService:
+
+  def __init__(
+      self,
+      classifier: ImageClassifier,
+      validator: ImageValidator,
+      clip_classifier: ClipClassifier
+  ):
+
+    self.classifier = classifier
+    self.clip_classifier = clip_classifier
+    self.validator = validator
+
+
+
+  def moderate(
+      self,
+      image: Image.Image
+  ) -> ImagePredictionResult:
+
+
+    self.validator.validate(image)
+
+
+    # =========================
+    # 1. NSFW MODEL
+    # =========================
+
+    nsfw_prediction = self.classifier.predict(image)
+
+
+    if (
+        nsfw_prediction.label.lower() == "nsfw"
+        and nsfw_prediction.score >= settings.NSFW_THRESHOLD
+    ):
+
+      nsfw_prediction.approved = False
+
+      nsfw_prediction.reason = "NSFW"
+
+      nsfw_prediction.detected_labels = [
+        nsfw_prediction.label
+      ]
+
+      return nsfw_prediction
+
+
+
+    # =========================
+    # 2. CLIP MODEL
+    # =========================
+
+    clip_prediction = (
+      self.clip_classifier.predict(image)
+    )
+
+
+    detected_forbidden = [
+
+      label
+
+      for label in clip_prediction.detected_labels
+
+      if label.lower()
+         in FORBIDDEN_IMAGE_LABELS
+
+    ]
+
+
+    if detected_forbidden:
+
+
+      return ImagePredictionResult(
+
+          label=clip_prediction.label,
+
+          score=clip_prediction.score,
+
+          approved=False,
+
+          reason="FORBIDDEN_CONTENT",
+
+          detected_labels=detected_forbidden
+
+      )
+
+
+
+    # =========================
+    # 3. NORMAL IMAGE
+    # =========================
+
+    return ImagePredictionResult(
+
+        label="normal",
+
+        score=1.0,
+
+        approved=True,
+
+        reason="OK",
+
+        detected_labels=[]
+
+    )

ai-moderation/app/services/text_moderation_service.py

@@ -1,26 +1,76 @@
+from app.exceptions.invalid_text_exception import InvalidTextException
+from app.moderation.profanity.detector import ProfanityDetector
 from app.ml.text_classifier import TextClassifier
 from app.ml.prediction_result import PredictionResult
+from app.config.settings import settings
 
 
 class TextModerationService:
 
-  TOXIC_THRESHOLD = 0.80
+  def __init__(
+      self,
+      classifier: TextClassifier,
+      profanity_detector: ProfanityDetector
+  ):
+
+    self.classifier = classifier
+    self.profanity_detector = profanity_detector
+
+
+  def moderate(
+      self,
+      text: str
+  ) -> PredictionResult:
+
+
+    if text is None or not text.strip():
+
+      raise InvalidTextException(
+          "Text is empty"
+      )
+
+
+    detected_words = (
+      self.profanity_detector.detect(text)
+    )
+
+
+    if detected_words:
+
+
+      return PredictionResult(
+
+          label="PROFANITY",
+
+          score=1.0,
+
+          approved=False,
+
+          raw_scores=[],
+
+          reason="PROFANITY",
+
+          detected_words=detected_words
+
+      )
 
-  def __init__(self):
-    self.classifier = TextClassifier()
 
-  def moderate(self, text: str) -> PredictionResult:
 
     prediction = self.classifier.predict(text)
 
+
+
     prediction.approved = (
-        prediction.score < self.TOXIC_THRESHOLD
+        prediction.score <
+        settings.TEXT_TOXIC_THRESHOLD
     )
 
+
     prediction.reason = (
       "OK"
       if prediction.approved
       else "TOXIC"
     )
 
+
     return prediction