Develop db_sync dockerfile and add input to build docker image on wor…

…flow
WattRex · Dec 18, 2023 · bc4461c · bc4461c
1 parent bc20ed0
commit bc4461c
Show file tree

Hide file tree

Showing 7 changed files with 117 additions and 39 deletions.
diff --git a/.github/workflows/03_deploy_production.yml b/.github/workflows/03_deploy_production.yml
@@ -49,6 +49,7 @@ jobs:
       docker-user: ${{ vars.DOCKER_USERNAME_RALDEA }}
       docker-user-arg: 69976 # wattrex
       docker-group-arg: 69976 # wattrex
+      docker-target: cycler_prod
 
   publish-docker-image-db-sync:
     name: Publish db-sync docker image
@@ -61,4 +62,8 @@ jobs:
       docker-repo-name: wattrex-cycler-db-sync
       is-develop: false
       docker-user: ${{ vars.DOCKER_USERNAME_RALDEA }}
+      docker-user-arg: 69976 # wattrex
+      docker-group-arg: 69976 # wattrex
+      docker-target: db_sync_prod
+
 
diff --git a/devops/cache_db/createCacheCyclerTables.sql b/devops/cache_db/createCacheCyclerTables.sql
@@ -1,5 +1,5 @@
--- Table Experiment --------------------------
-create table if not exists Experiment
+-- Table ExperimentCache --------------------------
+create table if not exists ExperimentCache
 (
     ExpID           mediumint       unsigned    not null,
     Name            varchar(30)                 not null,
@@ -12,7 +12,7 @@ create table if not exists Experiment
     BatID           mediumint       unsigned    not null,
     ProfID          mediumint       unsigned    not null,
 
-    constraint Experiment_pk_1
+    constraint ExperimentCache_pk_1
         primary key (ExpID)
 );
 
@@ -29,12 +29,12 @@ create table if not exists Alarm
     constraint Alarm_pk_1
         primary key (ExpID, AlarmID),
     constraint Alarm_fk_1
-        foreign key (ExpID) references Experiment (ExpID)
+        foreign key (ExpID) references ExperimentCache (ExpID)
 );
 
 
--- Table Status --------------------------
-create table if not exists Status
+-- Table StatusCache --------------------------
+create table if not exists StatusCache
 (
     StatusID        mediumint       unsigned    not null,
     ExpID           mediumint       unsigned    not null,
@@ -43,15 +43,15 @@ create table if not exists Status
     Status          enum ('OK', 'COMM_ERROR', 'INTERNAL_ERROR') not null,
     ErrorCode       smallint        unsigned    not null,
 
-    constraint Status_pk_1
+    constraint StatusCache_pk_1
         primary key (StatusID, ExpID),
-    constraint Status_fk_1
-        foreign key (ExpID) references Experiment (ExpID)
+    constraint StatusCache_fk_1
+        foreign key (ExpID) references ExperimentCache (ExpID)
 );
 
 
--- Table GenericMeasures --------------------------
-create table if not exists GenericMeasures
+-- Table GenericMeasuresCache --------------------------
+create table if not exists GenericMeasuresCache
 (
     ExpID           mediumint       unsigned    not null,
     MeasID          int             unsigned    not null,
@@ -62,28 +62,28 @@ create table if not exists GenericMeasures
     Power           int                         not null,
     PowerMode       enum ('DISABLE', 'WAIT', 'CC_MODE', 'CV_MODE', 'CP_MODE') not null,
 
-    constraint GenericMeasures_pk_1
+    constraint GenericMeasuresCache_pk_1
         primary key (ExpID, MeasID),
-    constraint GenericMeasures_fk_1
-        foreign key (ExpID) references Experiment (ExpID)
+    constraint GenericMeasuresCache_fk_1
+        foreign key (ExpID) references ExperimentCache (ExpID)
 );
 
 
--- Table ExtendedMeasures --------------------------
-create table if not exists ExtendedMeasures
+-- Table ExtendedMeasuresCache --------------------------
+create table if not exists ExtendedMeasuresCache
 (
     MeasID          int             unsigned    not null,
     ExpID           mediumint       unsigned    not null,
     UsedMeasID      mediumint       unsigned    not null,
     Value           mediumint                   not null,
 
-    constraint ExtendedMeasures_pk_1
+    constraint ExtendedMeasuresCache_pk_1
         primary key (ExpID, UsedMeasID, MeasID),
-    constraint ExtendedMeasures_fk_1
-        foreign key (ExpID, MeasID) references GenericMeasures (ExpID, MeasID)
+    constraint ExtendedMeasuresCache_fk_1
+        foreign key (ExpID, MeasID) references GenericMeasuresCache (ExpID, MeasID)
 );
 
 CREATE User 'basic_user'@'%' IDENTIFIED BY 'basic_user';
-GRANT SELECT, SHOW VIEW ON battery_experiments_manager_db.* TO 'basic_user'@'%';
+GRANT SELECT, SHOW VIEW ON wattrex_cache_db.* TO 'basic_user'@'%';
 -- ALTER USER 'basic_user'@'%' IDENTIFIED WITH mysql_native_password BY 'basic_user';
 FLUSH PRIVILEGES;
diff --git a/devops/cycler/Dockerfile.cycler b/devops/cycler/Dockerfile.cycler
@@ -13,16 +13,15 @@ RUN pip install --upgrade pip
 RUN pip install pytest
 ADD --chown=${UG_NAME}:${UG_NAME} ./devops/config_params_example.yaml ./devops/config_params.yaml
 ENV CONFIG_FILE_PATH=${APP_PATH}/devops/config_params.yaml
+ADD --chown=${UG_NAME}:${UG_NAME} ./devops/cycler ./devops/cycler
 
 FROM cycler_base as cycler_test
 ARG UPDATE_REQS=default
 ADD --chown=${UG_NAME}:${UG_NAME} ./code/cycler ./code/cycler
-ADD --chown=${UG_NAME}:${UG_NAME} ./devops/cycler ./devops/cycler
 RUN pip install -r ./code/cycler/requirements.txt
 
 FROM cycler_base as cycler_prod
 ARG UPDATE_REQS=default
 RUN pip install wattrex-battery-cycler
-ADD --chown=${UG_NAME}:${UG_NAME} ./devops/cycler ./devops/cycler
 
 CMD ["python", "./devops/cycler/run_cycler.py"]
diff --git a/devops/db_sync/Dockerfile.db_sync b/devops/db_sync/Dockerfile.db_sync
@@ -1,7 +1,26 @@
-FROM python:3.11-alpine3.17
-WORKDIR /db_sync
-RUN python -m pip install --upgrade pip
-ADD ./ ./
-RUN apk add build-base
-RUN pip install -r ./code/cycler/requirements.txt
-CMD ["python", "./code/cycler/src/wattrex_battery_cycler/mid/mid_sync/db_sync.py"]
+FROM python:3.10-bullseye as db_sync_base
+ARG USER=default
+ARG GROUP=default
+ENV U_ID=${USER}
+ENV G_UID=${GROUP}
+ENV UG_NAME=wattrex
+RUN addgroup --system --gid ${G_UID} ${UG_NAME} && adduser --system --gid ${G_UID} --uid ${U_ID} ${UG_NAME}
+USER ${UG_NAME}
+ENV APP_PATH=/cycler
+WORKDIR ${APP_PATH}
+ENV PATH="${PATH}:/home/${UG_NAME}/.local/bin"
+RUN pip install --upgrade pip
+ADD --chown=${UG_NAME}:${UG_NAME} ./devops/config_params_example.yaml ./devops/config_params.yaml
+ENV CONFIG_FILE_PATH=${APP_PATH}/devops/config_params.yaml
+ADD --chown=${UG_NAME}:${UG_NAME} ./devops/db_sync ./devops/db_sync
+
+FROM db_sync_base as db_sync_local
+ARG UPDATE_REQS=default
+ADD --chown=${UG_NAME}:${UG_NAME} ./code/db_sync ./code/db_sync
+RUN pip install -r ./code/db_sync/requirements.txt
+CMD ["python", "./devops/db_sync/run_db_sync.py"]
+
+FROM db_sync_base as db_sync_prod
+ARG UPDATE_REQS=default
+RUN pip install wattrex-cycler-db-sync
+CMD ["python", "./devops/db_sync/run_db_sync.py"]
diff --git a/devops/db_sync/run_db_sync.py b/devops/db_sync/run_db_sync.py
@@ -0,0 +1,39 @@
+#!/usr/bin/python3
+"""
+DB SYNC
+"""
+#######################        MANDATORY IMPORTS         #######################
+
+#######################         GENERIC IMPORTS          #######################
+import os
+import sys
+import threading
+
+#######################       THIRD PARTY IMPORTS        #######################
+
+#######################    SYSTEM ABSTRACTION IMPORTS    #######################
+from system_logger_tool import sys_log_logger_get_module_logger, SysLogLoggerC, Logger
+
+#######################       LOGGER CONFIGURATION       #######################
+cycler_logger = SysLogLoggerC(file_log_levels='./devops/db_sync/log_config.yaml',
+                              output_sub_folder='db_sync')
+log: Logger = sys_log_logger_get_module_logger(__name__)
+
+#######################          MODULE IMPORTS          #######################
+sys.path.append(os.path.dirname(__file__)+'/../')
+#from code.db_sync.src.wattrex_cycler_db_sync import DbSyncNodeC
+from wattrex_cycler_db_sync import DbSyncNodeC
+
+#######################          PROJECT IMPORTS         #######################
+
+#######################              ENUMS               #######################
+
+#######################             CLASSES              #######################
+
+#######################            FUNCTIONS             #######################
+
+if __name__ == '__main__':
+    working_flag_event : threading.Event = threading.Event()
+    working_flag_event.set()
+    db_sync_node = DbSyncNodeC(working_flag=working_flag_event)
+    db_sync_node.run()
diff --git a/devops/deploy.sh b/devops/deploy.sh
@@ -22,10 +22,7 @@ initial_deploy () {
     python3 -m pip install SCPI-sniffer
     mkdir -p "${REPO_ROOT_DIR}/log"
 
-    # python3 -m pip install scpi-sniffer
-    # sudo sh -c 'echo 250 > /proc/sys/fs/mqueue/msg_max'
-    docker compose ${DOCKER_COMPOSE_ARGS} up cache_db -d
-    #docker compose ${DOCKER_COMPOSE_ARGS} up cache_db db_sync -d
+    docker compose ${DOCKER_COMPOSE_ARGS} up cache_db db_sync -d
 
     check_sniffer "can"
     check_sniffer "scpi"
@@ -34,7 +31,6 @@ initial_deploy () {
 instance_new_cycler () {
     check_sniffer "can"
     check_sniffer "scpi"
-    # check_sniffer "scpi"
     export CYCLER_TARGET=cycler_prod
     docker compose ${DOCKER_COMPOSE_ARGS} build --build-arg UPDATE_REQS=$(date +%s) cycler
     docker compose ${DOCKER_COMPOSE_ARGS} run -d -e CSID=${1} --name wattrex_cycler_node_${1} cycler
@@ -127,7 +123,8 @@ fi
 # Check if the required files are present.
 required_file_list=("docker-compose.yml" ".cred.env" ".cred.yaml" "config_params.yaml"
                     "scpi/log_config.yaml" "cycler/log_config.yaml" "cu_manager/log_config.yaml"
-                    "can/log_config.yaml" "cache_db/createCacheCyclerTables.sql")
+                    "can/log_config.yaml" "cache_db/createCacheCyclerTables.sql"
+                    "db_sync/log_config.yaml" )
 for file in ${required_file_list}
 do
     file_path=${DEVOPS_DIR}/${file}
@@ -140,6 +137,14 @@ done
 case ${ARG1} in
     "")
         # echo "Initial Deploy"
+        export CYCLER_TARGET=db_sync_prod
+        docker compose ${DOCKER_COMPOSE_ARGS} pull db_sync
+        initial_deploy
+        ;;
+    "build")
+        # echo "Initial Deploy"
+        export CYCLER_TARGET=db_sync_local
+        docker compose ${DOCKER_COMPOSE_ARGS} build --build-arg UPDATE_REQS=$(date +%s) db_sync
         initial_deploy
         ;;
     "cycler")

diff --git a/devops/docker-compose.yml b/devops/docker-compose.yml
@@ -31,7 +31,7 @@ services:
       args:
         USER: ${USER_ID}
         GROUP: ${GROUP_ID}
-    image: wattrex-cycler-node
+    image: robertoaldea/wattrex-cycler-node
     restart: no
     user: ${USER_ID}:${GROUP_ID}
     ipc: host
@@ -50,11 +50,22 @@ services:
     build:
       context: ../
       dockerfile: ./devops/db_sync/Dockerfile.db_sync
-    image: wattrex-cycler-db-sync
+      target: ${CYCLER_TARGET}
+      args:
+        USER: ${USER_ID}
+        GROUP: ${GROUP_ID}
+    image: robertoaldea/wattrex-cycler-db-sync
     container_name: wattrex_cycler_db_sync
     restart: always
-    env_file:
-      - ./.cred.env
+    user: ${USER_ID}:${GROUP_ID}
+    ipc: host
+    volumes:
+      - ./db_sync/run_db_sync.py:/cycler/devops/db_sync/run_db_sync.py
+      - ./.cred.yaml:/cycler/devops/.cred.yaml
+      - ./db_sync/log_config.yaml:/cycler/devops/db_sync/log_config.yaml
+      - ../log:/cycler/log
+    networks:
+      - wattrex-net
     depends_on:
       cache_db:
         condition: service_healthy