From 48aae40737e55d0e17895facb9da56cfa5206f53 Mon Sep 17 00:00:00 2001 From: Arno Date: Thu, 7 Sep 2023 08:20:35 +0000 Subject: [PATCH] Initial commit --- .gitignore | 190 ++++++++++++++++++++ README.md | 51 ++++++ base_image/Dockerfile.python-3-8-fastapi | 46 +++++ deploy/Dockerfile | 9 + deploy/build.sh | 4 + deploy/deployment.yaml | 72 ++++++++ deploy/requirements_fastapi_mac.txt | 36 ++++ deploy/run.sh | 3 + html/index.html | 29 +++ main.py | 74 ++++++++ src/__init__.py | 0 src/websocket_manager/__init__.py | 1 + src/websocket_manager/connection_manager.py | 71 ++++++++ static/css/style.css | 3 + static/js/main.js | 86 +++++++++ 15 files changed, 675 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 base_image/Dockerfile.python-3-8-fastapi create mode 100644 deploy/Dockerfile create mode 100644 deploy/build.sh create mode 100644 deploy/deployment.yaml create mode 100644 deploy/requirements_fastapi_mac.txt create mode 100644 deploy/run.sh create mode 100644 html/index.html create mode 100644 main.py create mode 100644 src/__init__.py create mode 100644 src/websocket_manager/__init__.py create mode 100644 src/websocket_manager/connection_manager.py create mode 100644 static/css/style.css create mode 100644 static/js/main.js diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e23c600 --- /dev/null +++ b/.gitignore @@ -0,0 +1,190 @@ +# ---> Python +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# ---> macOS +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + diff --git a/README.md b/README.md new file mode 100644 index 0000000..bf3a9ae --- /dev/null +++ b/README.md @@ -0,0 +1,51 @@ +# python-3-8-fastapi + +# Base Image +Building the base image requires access to `docker.sciocatti.com` to pull the `docker.sciocatti.com/python:3.8-slim` image. Pulling this base image also requires access. If you do not have access to this registry, you can + +1. create your own image by replacing `docker.sciocatti.com/python:3.8-slim` with `python:3.8-slim` in `base_image/Dockerfile.python-3-8-fastapi`, and +1. building the dockerfile and using that as the base. +1. Optionally, you can push that to your own registry. + +If you do have access you can update the base image by +1. Updating `base_image/Dockerfile.python-3-8-fastapi`, and +1. Running the following: + ```bash + cd base_image + # Build + docker build -f Dockerfile.python-3-8-fastapi -t docker.sciocatti.com/python-3.8-fastapi:{{tag}} . + # Push + docker push docker.sciocatti.com/python-3.8-fastapi:{{tag}} + ``` + +## Installed libraries +- FastApi +- Python-dotenv +- Requests +- MySQL / MariaDB connector + +# Changing this from the template +This is almost ready to go as-is, you just need to make some tweaks: +1. Change the port in `deploy/run.sh`. By default the app will be running on port 64000 on your host. +1. Change the name of the output image in `deploy/build.sh` and potentially disable the push to the registry. +1. Change the K3S deployment details in `deploy/deployment.yaml`. This is made specifically for my cluster, so adapt for yours if using yours. + +# Running & Building +## Running +You only need docker to run this, and access to relevant images. +```bash +# In project root +bash run.sh +``` + +## Building +```bash +# In project root +bash build.sh +``` + +# Deploying +1. Update dockerfile. +1. Build the image. +1. Push the image. +1. Restart deployment in cluster. \ No newline at end of file diff --git a/base_image/Dockerfile.python-3-8-fastapi b/base_image/Dockerfile.python-3-8-fastapi new file mode 100644 index 0000000..e3210d4 --- /dev/null +++ b/base_image/Dockerfile.python-3-8-fastapi @@ -0,0 +1,46 @@ +FROM docker.sciocatti.com/python:3.8-slim + +RUN python3 -m venv /venv + +RUN . /venv/bin/activate && pip install --no-cache-dir \ + anyio==3.6.1 \ + asgiref==3.5.2 \ + certifi==2022.6.15 \ + charset-normalizer==2.0.12 \ + click==8.1.3 \ + dnspython==2.2.1 \ + email-validator==1.2.1 \ + fastapi==0.78.0 \ + h11==0.13.0 \ + httptools==0.4.0 \ + idna==3.3 \ + itsdangerous==2.1.2 \ + Jinja2==3.1.2 \ + MarkupSafe==2.1.1 \ + orjson==3.7.3 \ + pydantic==1.9.1 \ + python-dateutil==2.8.2 \ + python-dotenv==0.20.0 \ + python-multipart==0.0.5 \ + PyYAML==6.0 \ + requests==2.28.0 \ + six==1.16.0 \ + sniffio==1.2.0 \ + starlette==0.19.1 \ + typing-extensions==4.2.0 \ + ujson==5.3.0 \ + urllib3==1.26.9 \ + uvicorn==0.17.6 \ + uvloop==0.16.0 \ + watchgod==0.8.2 \ + websockets==10.3 \ + SQLAlchemy==1.4.42 \ + PyMySQL==1.0.2 \ + mysql-connector-python==8.0.31 \ + xmltodict==0.13.0 + +CMD . /venv/bin/activate && exec python + +# docker build -f Dockerfile.python-3-8-fastapi -t docker.sciocatti.com/python-3.8-fastapi:0.0.2 . +# docker run --name=bb_fastapi -d --restart=always -p 64000:15234 -v /root/home/iot/iot_bww_backend:/project docker.sciocatti.com/python-3.8-fastapi:0.0.1 /project/start.sh +# docker push docker.sciocatti.com/python-3.8-fastapi:0.0.2root@localhost:~/deployments/docker_images# \ No newline at end of file diff --git a/deploy/Dockerfile b/deploy/Dockerfile new file mode 100644 index 0000000..41aa1ef --- /dev/null +++ b/deploy/Dockerfile @@ -0,0 +1,9 @@ +FROM docker.sciocatti.com/python-3.8-fastapi:0.0.1 + +WORKDIR /project +COPY static/ static/ +COPY src/ src/ +COPY html/ html/ +COPY main.py main.py + +CMD . /venv/bin/activate && cd /project && exec python main.py \ No newline at end of file diff --git a/deploy/build.sh b/deploy/build.sh new file mode 100644 index 0000000..9bbd3d5 --- /dev/null +++ b/deploy/build.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +docker build -f Dockerfile -t docker.sciocatti.com/template:0.0.1 . +docker push docker.sciocatti.com/template:0.0.1 \ No newline at end of file diff --git a/deploy/deployment.yaml b/deploy/deployment.yaml new file mode 100644 index 0000000..8601c4c --- /dev/null +++ b/deploy/deployment.yaml @@ -0,0 +1,72 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: template + namespace: development + labels: + app: template +spec: + replicas: 1 + selector: + matchLabels: + app: template + strategy: + type: Recreate + template: + metadata: + labels: + app: template + spec: + nodeSelector: + kubernetes.io/arch: amd64 + imagePullSecrets: + - name: regcred + restartPolicy: Always + containers: + - name: template + image: docker.sciocatti.com/template:0.0.1 + imagePullPolicy: Always + ports: + - containerPort: 50001 + name: template-port + +--- +apiVersion: v1 +kind: Service +metadata: + name: template-service + namespace: development +spec: + selector: + app: template + ports: + - name: template-port + port: 3000 + targetPort: 50001 + protocol: TCP + +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: template-ingress + namespace: development + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod + traefik.ingress.kubernetes.io/router.entrypoints: web, websecure +spec: + rules: + - host: template.sciocatti.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: template-service + port: + number: 3000 + tls: + - hosts: + - template.sciocatti.com + secretName: template-sciocatti-com-tls \ No newline at end of file diff --git a/deploy/requirements_fastapi_mac.txt b/deploy/requirements_fastapi_mac.txt new file mode 100644 index 0000000..5287ee5 --- /dev/null +++ b/deploy/requirements_fastapi_mac.txt @@ -0,0 +1,36 @@ +anyio==3.6.1 +asgiref==3.5.2 +certifi==2022.6.15 +charset-normalizer==2.0.12 +click==8.1.3 +dnspython==2.2.1 +email-validator==1.2.1 +fastapi==0.78.0 +h11==0.13.0 +httptools==0.4.0 +idna==3.3 +itsdangerous==2.1.2 +Jinja2==3.1.2 +MarkupSafe==2.1.1 +mysql-connector-python==8.0.31 +orjson==3.7.3 +protobuf==3.20.1 +pydantic==1.9.1 +PyMySQL==1.0.2 +python-dateutil==2.8.2 +python-dotenv==0.20.0 +python-multipart==0.0.5 +PyYAML==6.0 +requests==2.28.0 +six==1.16.0 +sniffio==1.2.0 +SQLAlchemy==1.4.42 +starlette==0.19.1 +typing_extensions==4.2.0 +ujson==5.3.0 +urllib3==1.26.9 +uvicorn==0.17.6 +uvloop==0.16.0 +watchgod==0.8.2 +websockets==10.3 +xmltodict==0.13.0 diff --git a/deploy/run.sh b/deploy/run.sh new file mode 100644 index 0000000..7419df7 --- /dev/null +++ b/deploy/run.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker run -it --rm -p 64000:50001 -v $(pwd):/project docker.sciocatti.com/python-3.8-fastapi:0.0.1 bash -c 'cd /project && . /venv/bin/activate && exec python main.py' \ No newline at end of file diff --git a/html/index.html b/html/index.html new file mode 100644 index 0000000..43f9b19 --- /dev/null +++ b/html/index.html @@ -0,0 +1,29 @@ + + + + + + + Document + + + +
+ Server Status: Pending +
+ Some advice: +
+ + +
+
+ Socket Messages: +
+
+
+ + +
+ + + diff --git a/main.py b/main.py new file mode 100644 index 0000000..dd34092 --- /dev/null +++ b/main.py @@ -0,0 +1,74 @@ +import uvicorn +import logging +import requests +from fastapi import FastAPI +from fastapi import Response +from fastapi import WebSocket +from fastapi import WebSocketDisconnect +from fastapi.responses import FileResponse +from fastapi.staticfiles import StaticFiles +from pydantic import BaseModel + +from src.websocket_manager import ThreadedConnectionManager + +logging.basicConfig(format='%(asctime)s - %(levelname)s - %(pathname)s:%(lineno)d\r\n\t%(message)s', datefmt='%d-%b-%y %H:%M:%S', level=logging.INFO) + +PORT = 50001 + +# * * * * * * * * * * * * +# INIT FASTAPI CLASSES +# * * * * * * * * * * * * +app = FastAPI() +app.mount("/static", StaticFiles(directory="static"), name="static") + +# * * * * * * * * * * * * +# INIT CUSTOM CLASSES +# * * * * * * * * * * * * +manager = ThreadedConnectionManager() + +# * * * * * * * * * * * * +# WEBSOCKET ROUTES +# * * * * * * * * * * * * +@app.websocket("/ws/{client_id}") +async def websocket_endpoint(websocket: WebSocket, client_id: int): + await manager.connect(websocket) + try: + while True: + text = await websocket.receive_text() + print(text) + # await manager.send_personal_message(f"You wrote: {data}", websocket) + await manager.broadcast({"data": text}) + except WebSocketDisconnect: + manager.disconnect(websocket) + await manager.broadcast({"data": f"Client #{client_id} left the chat"}) + +# * * * * * * * * * * * * +# HTTP ROUTES +# * * * * * * * * * * * * +@app.get("/") +async def index(): + filename = f"html/index.html" + return FileResponse(filename) + +@app.get("/alive") +async def get_alive(): + return {} + +class AdviceBody(BaseModel): + topic: str + +@app.post("/advice") +async def post_advice(body: AdviceBody): + topic = body.topic + result = requests.get("https://api.adviceslip.com/advice/search/"+topic, timeout=20) + data = result.json() + if "slips" in data: + return data + return Response(content=data["message"]["text"], status_code=404, media_type="text/html", headers={}) + +def main(): + uvicorn.run(app, host="0.0.0.0", port=PORT) + manager.running = False + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/websocket_manager/__init__.py b/src/websocket_manager/__init__.py new file mode 100644 index 0000000..2175008 --- /dev/null +++ b/src/websocket_manager/__init__.py @@ -0,0 +1 @@ +from .connection_manager import ThreadedConnectionManager \ No newline at end of file diff --git a/src/websocket_manager/connection_manager.py b/src/websocket_manager/connection_manager.py new file mode 100644 index 0000000..3cd3d5d --- /dev/null +++ b/src/websocket_manager/connection_manager.py @@ -0,0 +1,71 @@ +import json +from typing import List, Union +from fastapi import WebSocket +import threading +import queue +import time +import asyncio + +class ThreadedConnectionManager(threading.Thread): + def __init__(self) -> None: + threading.Thread.__init__(self) + self.active_connections: List[WebSocket] = [] + self.message_queue: queue.Queue = queue.Queue(maxsize=1000) + self.running = True + self.setDaemon = True + self.start() + + def run(self): + while self.running: + time.sleep(0.1) + try: + message=self.message_queue.get(timeout=1) + asyncio.run(self.broadcast(message)) + except queue.Empty: + #Handle empty queue here + pass + + def add_message_to_broadcast_queue(self, message): + if type(message) == dict: + message = json.dumps(message) + self.message_queue.put(message) + + async def connect(self, websocket: WebSocket): + await websocket.accept() + self.active_connections.append(websocket) + + def disconnect(self, websocket: WebSocket): + self.active_connections.remove(websocket) + + async def send_personal_message(self, message: Union[str, dict], websocket: WebSocket): + if type(message) == dict: + message = json.dumps(message) + await websocket.send_text(message) + + async def broadcast(self, message: Union[str, dict]): + if type(message) == dict: + message = json.dumps(message) + for connection in self.active_connections: + try: + await connection.send_text(message) + except RuntimeError as e: + pass +class ConnectionManager: + def __init__(self): + self.active_connections: List[WebSocket] = [] + + async def connect(self, websocket: WebSocket): + await websocket.accept() + self.active_connections.append(websocket) + + def disconnect(self, websocket: WebSocket): + self.active_connections.remove(websocket) + + async def send_personal_message(self, message: str, websocket: WebSocket): + await websocket.send_text(message) + + async def broadcast(self, message: Union[str, dict]): + if type(message) == dict: + message = json.dumps(message) + for connection in self.active_connections: + await connection.send_text(message) \ No newline at end of file diff --git a/static/css/style.css b/static/css/style.css new file mode 100644 index 0000000..add21a5 --- /dev/null +++ b/static/css/style.css @@ -0,0 +1,3 @@ +body { + background-color: #a5a5a5; +} \ No newline at end of file diff --git a/static/js/main.js b/static/js/main.js new file mode 100644 index 0000000..0d0fbeb --- /dev/null +++ b/static/js/main.js @@ -0,0 +1,86 @@ +var client_id = Date.now(); +var protocol = window.location.protocol == "https:" ? "wss" : "ws"; +var ws; + +async function getServerStatus() { + let elem = document.querySelector("#serverStatus"); + elem.innerHTML = "Pending"; + // https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API/Using_Fetch + let url = "/alive"; + let response = await fetch(url, { + method: "GET", + headers: { + "Content-Type": "application/json", + } + }); + if (response.status != 200) { + elem.innerHTML = "Offline"; + return; + } + elem.innerHTML = "Online"; + let data = await response.json(); + return data; +} + +async function postAdvice() { + let topic = document.querySelector("#adviceTopic").value; + let elem = document.querySelector("#advice"); + let url = "/advice"; + let response = await fetch(url, { + method: "POST", + headers: { + "Content-type": "application/json" + }, + body: JSON.stringify({"topic": topic}) + }); + if (response.status == 404) { + elem.innerHTML = await response.text(); + return; + } + let data = await response.json(); + elem.innerHTML = data.slips[0].advice +} + +function connectSocket() { + ws = new WebSocket(`${protocol}://${window.location.host}/ws/${client_id}`); + ws.onopen = (event) => { + console.log(`Websocket opened -> ${client_id}`); + } + + ws.onclose = (event) => { + console.log("Websocket closed."); + setTimeout(connectSocket, 10000); + } + + ws.onerror = (event) => { + console.log("Websocket error."); + } + + ws.onmessage = (event) => { + console.log("New Message:"); + json_data = JSON.parse(event.data); + console.log(json_data); + let elem = document.querySelector("#socketMessages"); + let br = document.createElement("br"); + elem.appendChild(br); + elem.innerHTML += event.data; + } +} + +function sendMessage(message) { + ws.send(message); +} + +function sendSocketMessage() { + sendMessage(document.querySelector("#sendSocketMessage").value); +} + +async function onLoad() { + await getServerStatus(); + connectSocket(); + setInterval(async () => { + let alive = await getServerStatus(); + }, 30000); +} + +onLoad(); \ No newline at end of file