hotfix: add deploy and destroy in CI/CD, make all refactoring accordi…

…ng to merge comments

Makefile

@@ -1,6 +1,6 @@
 APP_PORT := 5039
 DOCKER_TAG := latest
-DOCKER_IMAGE := plates
+DOCKER_IMAGE := diunovidov_plates
 DVC_REMOTE_NAME := dvc_models_plates
 USERNAME := d.iunovidov
 

configs/config.yaml

@@ -2,3 +2,14 @@ content_process:
   dir_path: 'api_test_dir'
   dir_upload: 'uploaded_imgs'
 
+plate_model_parameters:
+  plate_checkpoint: "models/plate-model.onnx"
+  plate_img_width: 512
+  plate_img_height: 512
+
+ocr_model_parameters:
+  vocabular: "#&0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZÄÅÖÜĆČĐŠŽАБВГДЕЖЗИКЛМНОПРСТУФХЦЧШЭЮЯ"
+  ocr_checkpoint: "models/plate-ocr-model.onnx"
+  ocr_img_width: 416
+  ocr_img_height: 64
+  text_size: 10

deploy/ansible/deploy.yml

@@ -0,0 +1,55 @@
+- hosts: demo_host
+  tasks:
+    - name: create workdir
+      file:
+        path: '{{ playbook_dest }}'
+        state: directory
+        owner: '{{ ansible_user }}'
+      tags: [ 'prepare' ]
+
+    - name: create pull script
+      template:
+        dest: '{{ playbook_dest }}/pull.sh'
+        src: 'templates/pull.sh'
+        mode: 0755
+      tags: [ 'prepare' ]
+
+    - name: create run script
+      template:
+        dest: '{{ playbook_dest }}/run.sh'
+        src: 'templates/run.sh'
+        mode: 0755
+      tags: ['prepare']
+
+    - name: create destroy script
+      template:
+        dest: '{{ playbook_dest }}/destroy.sh'
+        src: 'templates/destroy.sh'
+        mode: 0755
+      tags: [ 'prepare' ]
+
+    - name: create clean script
+      template:
+        dest: '{{ playbook_dest }}/clean.sh'
+        src: 'templates/clean.sh'
+        mode: 0755
+      tags: [ 'prepare' ]
+
+    - name: pull docker image
+      command: '{{ playbook_dest }}/pull.sh'
+      tags: [ 'pull' ]
+
+    - name: stop and remove existing container
+      command: "{{ playbook_dest }}/destroy.sh"
+      ignore_errors: true
+      tags: [ 'deploy' ]
+
+    - name: run docker
+      command: "{{ playbook_dest }}/run.sh"
+      tags: [ 'deploy' ]
+
+#    - name: clean docker images and containers
+#      command: "{{ playbook_dest }}/clean.sh"
+#      ignore_errors: true
+#      tags: [ 'deploy' ]
+

deploy/ansible/destroy.yml

@@ -0,0 +1,32 @@
+- hosts: '{{ host }}'
+  tasks:
+    - name: create workdir
+      file:
+        path: '{{ playbook_dest }}'
+        state: directory
+        owner: '{{ ansible_user }}'
+      tags: [ 'prepare' ]
+
+    - name: create destroy script
+      template:
+        dest: '{{ playbook_dest }}/destroy.sh'
+        src: 'templates/destroy.sh'
+        mode: 0755
+      tags: [ 'prepare' ]
+
+    - name: create clean script
+      template:
+        dest: '{{ playbook_dest }}/clean.sh'
+        src: 'templates/clean.sh'
+        mode: 0755
+      tags: [ 'prepare' ]
+
+    - name: stop and remove existing container
+      command: "{{ playbook_dest }}/destroy.sh"
+      ignore_errors: true
+      tags: [ 'deploy' ]
+
+    - name: clean docker images and containers
+      command: "{{ playbook_dest }}/clean.sh"
+      ignore_errors: true
+      tags: [ 'deploy' ]

deploy/ansible/inventory.ini

@@ -0,0 +1,12 @@
+[demo_host]
+91.206.15.25
+
+[all:vars]
+container_name=diunovidov_ocr_service
+
+[demo_host:vars]
+playbook_dest=/home/d.iunovidov/ocr_service
+ansible_user=d.iunovidov
+service_port=5039
+ansible_ssh_private_key_file=~/.ssh/id_rsa
+

deploy/ansible/templates/clean.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -ue
+
+docker image prune -af
+docker container prune -af

deploy/ansible/templates/destroy.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -ue
+
+CONTAINER_NAME={{ container_name }}
+
+docker stop ${CONTAINER_NAME} || true
+
+docker rm -f ${CONTAINER_NAME} || true

deploy/ansible/templates/pull.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -ue
+
+docker login -u {{ docker_registry_user }} -p {{ docker_registry_password }} {{ docker_registry }}
+
+docker pull {{ docker_image }}:{{ docker_tag }}

deploy/ansible/templates/run.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -ue
+
+IMAGE={{ docker_image }}:{{ docker_tag }}
+
+docker run \
+    -d \
+    -p {{ service_port }}:5000 \
+    --name={{ container_name }} \
+    --restart always \
+    ${IMAGE}

models.dvc

@@ -1,5 +1,5 @@
 outs:
-- md5: f610bd1032fd0ff42d31c8e07947f697.dir
+- md5: 3054be2008ea3ed845b5aa02854ee19e.dir
   size: 108203922
   nfiles: 2
   hash: md5

src/containers/containers.py

@@ -2,18 +2,31 @@
 from dependency_injector import containers, providers
 
 from src.services.plate_process import ProcessPlate, Storage
+from src.services.preprocess_utils import OCRPredictor, PlatePredictor
 
 
 class Container(containers.DeclarativeContainer):
     """Container for DPI plates"""
     config = providers.Configuration()
 
-    store = providers.Factory(
+    store = providers.Singleton(
         Storage,
         config=config.content_process,
     )
 
+    plate_predictor = providers.Singleton(
+        PlatePredictor,
+        config=config.plate_model_parameters,
+    )
+
+    ocr_predictor = providers.Singleton(
+        OCRPredictor,
+        config=config.ocr_model_parameters,
+    )
+
     content_process = providers.Singleton(
         ProcessPlate,
         storage=store.provider(),
+        plate_predictor=plate_predictor.provider(),
+        ocr_predictor=ocr_predictor.provider(),
     )

src/routes/plates.py

@@ -4,7 +4,6 @@
 import numpy as np
 from dependency_injector.wiring import Provide, inject
 from fastapi import Depends, File, UploadFile
-from PIL import Image
 
 from src.containers.containers import Container
 from src.routes.routers import router
@@ -24,7 +23,9 @@ def get_content(
     :return: dict with content
     """
     return {
-        "content": storage.get(content_id),
+        "code": 200,
+        "predictions": storage.get(content_id),
+        "error": None,
     }
 
 
@@ -45,20 +46,16 @@ def process_content(
     :param content_process: container with process functionality
     :return: dictionary with results in json format
     """
-    try:
-        image_data = content_image.file.read()
-    except Exception:
-        return {"message": "There was an error uploading the file"}
-    finally:
-        content_image.file.close()
+    image_data = content_image.file.read()
+    content_image.file.close()
 
     image = cv2.imdecode(np.frombuffer(image_data, np.uint8), cv2.IMREAD_COLOR)
-    Image.fromarray(image)
     str_process = content_process.process(
         image,
         str(content_image.filename),
     )
     return {
-        "message": f"Successfully uploaded {content_image.filename}",
-        "scores": str_process,
+        "code": 200,
+        "predictions": str_process,
+        "error": None,
     }

src/services/plate_process.py

@@ -7,7 +7,7 @@
 
 import numpy as np
 
-from src.services.preprocess_utils import get_crop, get_ocr
+from src.services.preprocess_utils import PlatePredictor, OCRPredictor
 
 
 class Storage:
@@ -53,8 +53,16 @@ class ProcessPlate:
     """Class for storing processed"""
     status = "Start process image first"
 
-    def __init__(self, storage: Storage):
+    def __init__(
+        self,
+        storage: Storage,
+        plate_predictor: PlatePredictor,
+        ocr_predictor: OCRPredictor
+    ):
         self._storage = storage
+        self.plate_predictor = plate_predictor
+        self.ocr_predictor = ocr_predictor
+        print("finish process plate init")
 
     def process(self, image: np.ndarray, content_id: str) -> dict:
         """
@@ -63,12 +71,17 @@ def process(self, image: np.ndarray, content_id: str) -> dict:
         :param content_id: id of input image
         :return: dictionary with results in json format
         """
+        print("Call", content_id)
         json_responce = self._storage.get(content_id)
         if json_responce == self.status:
             # 1st stage - get plate crop
-            image_plate_crop, bbox_xxyy = get_crop(image)
+            image_plate_crop, bbox_xxyy = self.plate_predictor.predict(
+                image,
+            )
             # 2nd stage - get OCR plate number
-            plate_ocr = get_ocr(image_plate_crop)
+            plate_ocr = self.ocr_predictor.predict(
+                image_plate_crop
+            )
             # 3rd stage - save data
             json_responce = {
                 "plates": [