Draft: #1279 Develop elastic backend module

.ci/static_analysis_check.sh

@@ -20,14 +20,14 @@ pflake8 --version
 for folder in api/http api/python bingo/bingo-elastic/python api/tests/integration utils/indigo-service/backend/service
 do
   cd ${folder}
-  isort --check .
+  isort --check --profile=black .
   black --check .
   pflake8 .
   cd -
 done
 
 mypy --version
-for folder in api/http api/python bingo/bingo-elastic/python utils/indigo-service/backend/service
+for folder in api/http api/python bingo/bingo-elastic/python utils/indigo-service/backend/service/elastic utils/indigo-service/backend/service/postgres
 do
     cd ${folder}
     export PYTHONPATH=${PWD}

.github/workflows/indigo-ci.yaml

@@ -1319,15 +1319,15 @@ jobs:
           name: indigo-python
           path: utils/indigo-service/backend/lib/
       - name: Build
-        run: docker build -f ./utils/indigo-service/backend/Dockerfile -t epmlsop/indigo-service:latest ./utils/indigo-service/backend
+        run: docker build -f ./utils/indigo-service/backend/service/postgres/Dockerfile -t epmlsop/indigo-service:latest ./utils/indigo-service/backend/service/postgres
       - name: Test Imago
         run: |
           docker run  --rm=true -d -p 8080:80 --name=indigo_service epmlsop/indigo-service:latest
           sleep 10
           docker logs indigo_service
           docker ps
           export INDIGO_SERVICE_URL=http://localhost:8080/v2
-          python3 utils/indigo-service/backend/service/tests/api/imago_test.py
+          python3 utils/indigo-service/backend/service/postgres/service/tests/api/imago_test.py
           docker logs indigo_service
           docker stop indigo_service
 # TODO: add indigo tests

.gitignore

@@ -73,7 +73,7 @@ api/python/indigo/lib
 *.nupkg
 *.whl
 bin
-lib
+!lib
 obj
 TestResults
 cmake-build-debug

utils/indigo-service/.env

@@ -0,0 +1 @@
+COMPOSE_PROJECT_NAME=indigo-service

utils/indigo-service/MANUALY_DEPLOY_CLOUD.md

@@ -0,0 +1,60 @@
+# How to Deploy Indigo-Service in the Cloud
+
+Follow these steps to deploy indigo-service in a cloud environment.
+
+## Prerequisites
+
+1. On the remote machine, update your installed packages.
+2. Install Docker (For example, on AWS
+   Linux: [click](https://www.digitalocean.com/community/tutorials/how-to-install-and-use-docker-on-centos-7))
+
+## Deployment Steps
+
+1. Set up a local environment variable `HOST_IP` on the machine where you want to deploy the service. Use the following
+   commands based on your OS:
+
+   For Windows: `set HOST_IP=<ip address of instance>`
+
+   For MacOS/Unix: `export HOST_IP=<ip address of instance>`
+
+2. In the `ui` folder of the frontend part, find the `.env` file and set up the following variables:
+
+   ```
+   REACT_APP_API_POSTGRES=http://<ip address of instance>:8080/v2
+   REACT_APP_API_ELASTIC=http://<ip address of instance>:8080/v3
+   ```
+
+3. Build the `ui` folder using `yarn` by running the following command:
+
+   ```
+   yarn build
+   ```
+
+   **Note**: Don't forget to remove the old build folder if it exists.
+
+4. Zip the `indigo-service` folder.
+
+5. Upload the zipped file to your instance. You can use
+   the [scp](https://www.geeksforgeeks.org/scp-command-in-linux-with-examples/) command or apps
+   like [FileZilla](https://filezilla-project.org/).
+
+6. Unzip the uploaded file on the remote machine by running:
+
+   ```
+   unzip indigo-service.zip
+   ```
+
+7. In the root directory of `indigo-service`, use Docker to build images and run them with the following commands:
+
+   ```
+   docker-compose build --no-cache
+   docker-compose up -d
+   ```
+
+8. Check the result in your browser by navigating to:
+
+   ```
+   http://<ip-address>:8080
+   ```
+
+Now you have successfully deployed indigo-service in the cloud!

utils/indigo-service/MANUALY_DEPLOY_LOCAL.md

@@ -0,0 +1,40 @@
+# How to Deploy Indigo-Service Locally
+
+Follow these steps to deploy indigo-service on your local machine.
+
+## Deployment Steps
+
+1. Set up a local environment variable `HOST_IP` on your machine. Use the following commands based on your OS:
+
+   For Windows: `set HOST_IP=localhost`
+
+   For MacOS/Unix: `export HOST_IP=localhost`
+
+2. In the `ui` folder of the frontend part, find the `.env` file and set up the following variables:
+
+   ```
+   REACT_APP_API_POSTGRES=http://localhost:8080/v2
+   REACT_APP_API_ELASTIC=http://localhost:8080/v3
+   ```
+
+3. Build the `ui` folder using `yarn` by running the following command:
+
+   ```
+   yarn build
+   ```
+
+   **Note**: Don't forget to remove the old build folder if it exists.
+
+4. Open a command prompt or terminal at the root of `indigo-service`, and use Docker to build images and run them with
+   the following commands:
+
+   ```
+   docker-compose build --no-cache
+   docker-compose up -d
+   ```
+
+Now you have successfully deployed indigo-service on your local machine!
+
+## Useful Links
+
+- [How to install yarn](https://classic.yarnpkg.com/lang/en/docs/install/#windows-stable)

utils/indigo-service/backend/service/elastic/Dockerfile

@@ -0,0 +1,50 @@
+FROM ubuntu:22.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt update && \
+    apt upgrade -y
+RUN apt-get  -y install software-properties-common
+RUN add-apt-repository -y ppa:deadsnakes/ppa
+RUN apt install -y --no-install-recommends \
+        ca-certificates \
+        curl \
+        libfreetype6-dev \
+        libfontconfig1-dev \
+        python3.9 \
+        python3-pip \
+        python3-wheel \
+        python3-setuptools \
+        redis-server \
+        supervisor \
+        unzip \
+        uwsgi \
+        uwsgi-plugin-python3
+
+# Setup celery
+COPY ./conf/celeryd.conf /etc/default/celeryd
+RUN useradd -ms /bin/bash celery || echo "User already exists."
+RUN chmod 640 /etc/default/celeryd
+COPY ./conf/celery.auto.conf /etc/supervisor/conf.d/
+# Setup redis
+COPY ./conf/redis.auto.conf /etc/supervisor/conf.d/
+
+# Setup gunicorn
+COPY ./conf/gunicorn.auto.conf /etc/supervisor/conf.d/
+
+COPY ./service/v3/ /srv/api/v3/
+COPY ./service/*.py /srv/api/
+COPY ./service/v3/common/*.py /srv/api/
+COPY ./service/.env /srv/api/
+RUN mkdir -p srv/api/upload
+# Install python dependencies using pip
+COPY ./service/requirements.txt /opt/
+RUN pip3 install --no-cache-dir -r /opt/requirements.txt
+
+# Clean
+RUN apt autoremove -y && \
+    rm -rf /opt/* /var/lib/apt/lists/*
+
+EXPOSE 80
+WORKDIR /srv/api
+CMD supervisord -n

utils/indigo-service/backend/service/elastic/conf/celery.auto.conf

@@ -0,0 +1,12 @@
+[program:celery]
+directory=/srv/api
+user=root
+
+command=celery -A v3.celery_app worker -l INFO
+autostart=true
+autorestart=true
+redirect_stderr=true
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+
+killasgroup=true

utils/indigo-service/backend/service/elastic/conf/celeryd.conf

@@ -0,0 +1,38 @@
+ENABLED="true"
+
+# Names of nodes to start
+#   most will only start one node:
+#CELERYD_NODES="worker"
+#   but you can also start multiple and configure settings
+#   for each in CELERYD_OPTS (see `celery multi --help` for examples).
+#CELERYD_NODES="worker1 worker2 worker3"
+
+# Absolute or relative path to the 'celery' command:
+#CELERY_BIN="/usr/local/bin/celery"
+#CELERY_BIN="/virtualenvs/def/bin/celery"
+
+# App instance to use
+# comment out this line if you don't use an app
+CELERY_APP="v3.celery_app"
+# or fully qualified:
+#CELERY_APP="proj.tasks:app"
+
+# Where to chdir at start.
+CELERYD_CHDIR="/srv/api/"
+
+# Extra command-line arguments to the worker
+#CELERYD_OPTS="--time-limit=300 --concurrency=8"
+
+# %N will be replaced with the first part of the nodename.
+CELERYD_LOG_FILE="/var/log/celery/%N.log"
+CELERYD_PID_FILE="/var/run/celery/%N.pid"
+
+# Workers should run as an unprivileged user.
+#   You need to create this user manually (or you can choose
+#   a user/group combination that already exists, e.g. nobody).
+CELERYD_USER="celery"
+CELERYD_GROUP="celery"
+
+# If enabled pid and log directories will be created if missing,
+# and owned by the userid/group configured.
+CELERY_CREATE_DIRS=1

utils/indigo-service/backend/service/elastic/conf/gunicorn.auto.conf

@@ -0,0 +1,12 @@
+[program:gunicorn]
+directory=/srv/api
+user=root
+
+command=/bin/bash -c "gunicorn --bind 0.0.0.0:80 --workers=$(nproc) --timeout=0 app:app"
+autostart=true
+autorestart=true
+
+redirect_stderr=true
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+killasgroup=true

utils/indigo-service/backend/service/elastic/service/.env

@@ -0,0 +1,5 @@
+scheme=http
+host=elastic
+port=9200
+elastic_name=elastic
+path_to_upload=./upload

utils/indigo-service/backend/service/elastic/service/.gitignore

@@ -0,0 +1,3 @@
+/.idea/
+/venv/
+/python-elastic.iml

utils/indigo-service/backend/service/elastic/service/app.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+
+import json
+import logging
+import sys
+
+import urllib3 as urllib3
+from dotenv import load_dotenv  # type: ignore
+from flask import Flask
+from flask_cors import CORS  # type: ignore
+from v3.libraries_api import libraries_api
+from werkzeug import run_simple
+
+# Flask set-up
+load_dotenv()
+app = Flask(__name__)
+CORS(app)
+app.register_blueprint(libraries_api, url_prefix="/v3/libraries")
+urllib3.disable_warnings()
+
+logging.basicConfig(
+    stream=sys.stdout,
+    format="[%(asctime)s: %(levelname)-8s/%(filename)s:%(lineno)d]  %(message)s",
+    level=app.config.get("LOG_LEVEL"),
+)
+
+
+@app.errorhandler(Exception)
+def handle_exception(e):
+    response = e.get_response()
+    response.data = json.dumps(
+        {
+            "code": e.code,
+            "name": e.name,
+            "description": e.description,
+        }
+    )
+    response.content_type = "application/json"
+    return response, 500
+
+
+if __name__ == "__main__":
+    run_simple(
+        "0.0.0.0",
+        80,
+        app,
+        use_reloader=True,
+        use_debugger=True,
+        use_evalex=True,
+    )

utils/indigo-service/backend/service/elastic/service/clean-up-script.py

@@ -0,0 +1,29 @@
+import os
+
+from indigo import Indigo
+import sys
+import re
+
+if __name__ == '__main__':
+    indigo = Indigo()
+    try:
+        fileName = sys.argv[1]
+    except Exception:
+        raise ValueError("FileName should be provided")
+
+    match = re.search(r'([\w_-]+)\.\w+$', fileName)
+    if match:
+        filename_without_ext = match.group(1)
+        newName = f"clean/{filename_without_ext}_clean.sdf"
+        os.makedirs(os.path.dirname(newName), exist_ok=True)
+        with open(newName, "w") as f:
+            sdf = indigo.iterateSDFile(fileName)
+            i = 0
+            for mol in sdf:
+                if indigo.check(mol.rawData()) == "{}":
+                    f.write(mol.rawData())
+                    f.write("\n$$$$\n")
+                    print(i)
+                    i += 1
+    else:
+        raise Exception("Cannot retrieve file name.")

utils/indigo-service/backend/service/elastic/service/requirements.txt

@@ -0,0 +1,12 @@
+celery==5.2.7
+redis==4.5.5
+aenum==3.1.11
+bingo_elastic==1.14.0rc1
+elasticsearch==7.16.2
+epam.indigo==1.14.0.rc1
+Flask==2.2.3
+Flask_Cors==3.0.10
+python-dotenv==1.0.0
+urllib3==1.26.14
+gunicorn==20.1.0
+Werkzeug==2.2.2

utils/indigo-service/backend/service/elastic/service/tests/docker-compose.yml

@@ -0,0 +1,14 @@
+version: "3.4"
+
+services:
+  elastic:
+    image: elasticsearch:8.8.1
+    ports:
+      - 9200:9200
+    environment:
+      - discovery.type=single-node
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+      - xpack.license.self_generated.type=trial
+      - xpack.security.enabled=false
+      - xpack.security.http.ssl.enabled=false
+      - xpack.security.transport.ssl.enabled=false

utils/indigo-service/backend/service/elastic/service/tests/tests.py

@@ -0,0 +1,13 @@
+import unittest
+
+from bingo_elastic.elastic import ElasticRepository, IndexName
+from bingo_elastic.model import helpers
+
+
+class GeneratorTest(unittest.TestCase):
+    def test_generator_exception(self):
+        sdf = helpers.iterate_sdf("data/big_test.sdf")
+        ElasticRepository(IndexName.BINGO_CUSTOM, host="localhost", port=9200).index_records(records=sdf)
+
+if __name__ == '__main__':
+    unittest.main()